- CONDA_PY=2.7
- CONDA_PY=3.6
- CONDA_PY=3.7
+ - CONDA_PY=3.8
global:
- PYSAM_LINKING_TEST=1
- TWINE_USERNAME=grepall
compilation options. Especially for OS X this will potentially save a
lot of trouble.
-The current version of pysam wraps 3rd-party code from htslib-1.9, samtools-1.9, and bcftools-1.9.
+The current version of pysam wraps 3rd-party code from htslib-1.10.2, samtools-1.10, and bcftools-1.10.2.
Pysam is available through `pypi
<https://pypi.python.org/pypi/pysam>`_. To install, type::
-----------------------------------------------------------------------------
+LICENSE FOR VariantKey (https://github.com/Genomicsplc/variantkey)
+
+The MIT License
+
+Copyright (c) 2017-2018 GENOMICS plc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
memset(bca->rev_mqs,0,sizeof(int)*bca->nqual);
if ( call->ADF ) memset(call->ADF,0,sizeof(int32_t)*(call->n+1)*B2B_MAX_ALLELES);
if ( call->ADR ) memset(call->ADR,0,sizeof(int32_t)*(call->n+1)*B2B_MAX_ALLELES);
+ if ( call->SCR ) memset(call->SCR,0,sizeof(*call->SCR)*(call->n+1));
}
/*
memset(r->qsum,0,sizeof(float)*4);
memset(r->anno,0,sizeof(double)*16);
memset(r->p,0,sizeof(float)*25);
+ r->SCR = 0;
if (ref_base >= 0) {
ref4 = seq_nt16_int[ref_base];
if (q > 63) q = 63;
if (q < 4) q = 4; // MQ=0 reads count as BQ=4
bca->bases[n++] = q<<5 | (int)bam_is_rev(p->b)<<4 | b;
+ if ( bca->fmt_flag&(B2B_INFO_SCR|B2B_FMT_SCR) && PLP_HAS_SOFT_CLIP(p->cd.i) ) r->SCR++;
// collect annotations
if (b < 4)
{
// collect for bias tests
if ( baseQ > 59 ) baseQ = 59;
if ( mapQ > 59 ) mapQ = 59;
- int len, pos = get_position(p, &len);
- int epos = (double)pos/(len+1) * bca->npos;
+ int len, epos = 0;
+ if ( bca->fmt_flag & (B2B_INFO_RPB|B2B_INFO_VDB) )
+ {
+ int pos = get_position(p, &len);
+ epos = (double)pos/(len+1) * bca->npos;
+ }
int ibq = baseQ/60. * bca->nqual;
int imq = mapQ/60. * bca->nqual;
if ( bam_is_rev(p->b) ) bca->rev_mqs[imq]++;
call->DP4[4*i+3] = calls[i].anno[3];
}
}
+ if ( call->SCR )
+ {
+ for (i=0; i<n; i++)
+ {
+ call->SCR[0] += calls[i].SCR;
+ call->SCR[1+i] = calls[i].SCR;
+ }
+ }
if ( call->ADF )
{
assert( call->n_alleles<=B2B_MAX_ALLELES ); // this is always true for SNPs and so far for indels as well
// calc_chisq_bias("XMQ", call->bcf_hdr->id[BCF_DT_CTG][call->tid].key, call->pos, bca->ref_mq, bca->alt_mq, bca->nqual);
// calc_chisq_bias("XBQ", call->bcf_hdr->id[BCF_DT_CTG][call->tid].key, call->pos, bca->ref_bq, bca->alt_bq, bca->nqual);
- call->mwu_pos = calc_mwu_bias(bca->ref_pos, bca->alt_pos, bca->npos);
+ if ( bca->fmt_flag & B2B_INFO_RPB )
+ call->mwu_pos = calc_mwu_bias(bca->ref_pos, bca->alt_pos, bca->npos);
call->mwu_mq = calc_mwu_bias(bca->ref_mq, bca->alt_mq, bca->nqual);
call->mwu_bq = calc_mwu_bias(bca->ref_bq, bca->alt_bq, bca->nqual);
call->mwu_mqs = calc_mwu_bias(bca->fwd_mqs, bca->rev_mqs, bca->nqual);
#if CDF_MWU_TESTS
- call->mwu_pos_cdf = calc_mwu_bias_cdf(bca->ref_pos, bca->alt_pos, bca->npos);
+ // CDF version of MWU tests is not calculated by default
+ if ( bca->fmt_flag & B2B_INFO_RPB )
+ call->mwu_pos_cdf = calc_mwu_bias_cdf(bca->ref_pos, bca->alt_pos, bca->npos);
call->mwu_mq_cdf = calc_mwu_bias_cdf(bca->ref_mq, bca->alt_mq, bca->nqual);
call->mwu_bq_cdf = calc_mwu_bias_cdf(bca->ref_bq, bca->alt_bq, bca->nqual);
call->mwu_mqs_cdf = calc_mwu_bias_cdf(bca->fwd_mqs, bca->rev_mqs, bca->nqual);
#endif
- call->vdb = calc_vdb(bca->alt_pos, bca->npos);
+ if ( bca->fmt_flag & B2B_INFO_VDB )
+ call->vdb = calc_vdb(bca->alt_pos, bca->npos);
return 0;
}
if ( fmt_flag&B2B_INFO_DPR )
bcf_update_info_int32(hdr, rec, "DPR", bc->ADF, rec->n_allele);
}
+ if ( fmt_flag&B2B_INFO_SCR )
+ bcf_update_info_int32(hdr, rec, "SCR", bc->SCR, 1);
float tmpf[16];
for (i=0; i<16; i++) tmpf[i] = bc->anno[i];
if ( fmt_flag&B2B_FMT_DPR )
bcf_update_format_int32(hdr, rec, "DPR", bc->ADF+B2B_MAX_ALLELES, rec->n_sample*rec->n_allele);
}
+ if ( fmt_flag&B2B_FMT_SCR )
+ bcf_update_format_int32(hdr, rec, "SCR", bc->SCR+1, rec->n_sample);
return 0;
}
memset(bca->rev_mqs,0,sizeof(int)*bca->nqual);
if ( call->ADF ) memset(call->ADF,0,sizeof(int32_t)*(call->n+1)*B2B_MAX_ALLELES);
if ( call->ADR ) memset(call->ADR,0,sizeof(int32_t)*(call->n+1)*B2B_MAX_ALLELES);
+ if ( call->SCR ) memset(call->SCR,0,sizeof(*call->SCR)*(call->n+1));
}
/*
memset(r->qsum,0,sizeof(float)*4);
memset(r->anno,0,sizeof(double)*16);
memset(r->p,0,sizeof(float)*25);
+ r->SCR = 0;
if (ref_base >= 0) {
ref4 = seq_nt16_int[ref_base];
if (q > 63) q = 63;
if (q < 4) q = 4; // MQ=0 reads count as BQ=4
bca->bases[n++] = q<<5 | (int)bam_is_rev(p->b)<<4 | b;
+ if ( bca->fmt_flag&(B2B_INFO_SCR|B2B_FMT_SCR) && PLP_HAS_SOFT_CLIP(p->cd.i) ) r->SCR++;
// collect annotations
if (b < 4)
{
// collect for bias tests
if ( baseQ > 59 ) baseQ = 59;
if ( mapQ > 59 ) mapQ = 59;
- int len, pos = get_position(p, &len);
- int epos = (double)pos/(len+1) * bca->npos;
+ int len, epos = 0;
+ if ( bca->fmt_flag & (B2B_INFO_RPB|B2B_INFO_VDB) )
+ {
+ int pos = get_position(p, &len);
+ epos = (double)pos/(len+1) * bca->npos;
+ }
int ibq = baseQ/60. * bca->nqual;
int imq = mapQ/60. * bca->nqual;
if ( bam_is_rev(p->b) ) bca->rev_mqs[imq]++;
call->DP4[4*i+3] = calls[i].anno[3];
}
}
+ if ( call->SCR )
+ {
+ for (i=0; i<n; i++)
+ {
+ call->SCR[0] += calls[i].SCR;
+ call->SCR[1+i] = calls[i].SCR;
+ }
+ }
if ( call->ADF )
{
assert( call->n_alleles<=B2B_MAX_ALLELES ); // this is always true for SNPs and so far for indels as well
// calc_chisq_bias("XMQ", call->bcf_hdr->id[BCF_DT_CTG][call->tid].key, call->pos, bca->ref_mq, bca->alt_mq, bca->nqual);
// calc_chisq_bias("XBQ", call->bcf_hdr->id[BCF_DT_CTG][call->tid].key, call->pos, bca->ref_bq, bca->alt_bq, bca->nqual);
- call->mwu_pos = calc_mwu_bias(bca->ref_pos, bca->alt_pos, bca->npos);
+ if ( bca->fmt_flag & B2B_INFO_RPB )
+ call->mwu_pos = calc_mwu_bias(bca->ref_pos, bca->alt_pos, bca->npos);
call->mwu_mq = calc_mwu_bias(bca->ref_mq, bca->alt_mq, bca->nqual);
call->mwu_bq = calc_mwu_bias(bca->ref_bq, bca->alt_bq, bca->nqual);
call->mwu_mqs = calc_mwu_bias(bca->fwd_mqs, bca->rev_mqs, bca->nqual);
#if CDF_MWU_TESTS
- call->mwu_pos_cdf = calc_mwu_bias_cdf(bca->ref_pos, bca->alt_pos, bca->npos);
+ // CDF version of MWU tests is not calculated by default
+ if ( bca->fmt_flag & B2B_INFO_RPB )
+ call->mwu_pos_cdf = calc_mwu_bias_cdf(bca->ref_pos, bca->alt_pos, bca->npos);
call->mwu_mq_cdf = calc_mwu_bias_cdf(bca->ref_mq, bca->alt_mq, bca->nqual);
call->mwu_bq_cdf = calc_mwu_bias_cdf(bca->ref_bq, bca->alt_bq, bca->nqual);
call->mwu_mqs_cdf = calc_mwu_bias_cdf(bca->fwd_mqs, bca->rev_mqs, bca->nqual);
#endif
- call->vdb = calc_vdb(bca->alt_pos, bca->npos);
+ if ( bca->fmt_flag & B2B_INFO_VDB )
+ call->vdb = calc_vdb(bca->alt_pos, bca->npos);
return 0;
}
if ( fmt_flag&B2B_INFO_DPR )
bcf_update_info_int32(hdr, rec, "DPR", bc->ADF, rec->n_allele);
}
+ if ( fmt_flag&B2B_INFO_SCR )
+ bcf_update_info_int32(hdr, rec, "SCR", bc->SCR, 1);
float tmpf[16];
for (i=0; i<16; i++) tmpf[i] = bc->anno[i];
if ( fmt_flag&B2B_FMT_DPR )
bcf_update_format_int32(hdr, rec, "DPR", bc->ADF+B2B_MAX_ALLELES, rec->n_sample*rec->n_allele);
}
+ if ( fmt_flag&B2B_FMT_SCR )
+ bcf_update_format_int32(hdr, rec, "SCR", bc->SCR+1, rec->n_sample);
return 0;
}
#define B2B_INFO_AD (1<<9)
#define B2B_INFO_ADF (1<<10)
#define B2B_INFO_ADR (1<<11)
+#define B2B_INFO_SCR (1<<12)
+#define B2B_FMT_SCR (1<<13)
+#define B2B_INFO_VDB (1<<14)
+#define B2B_INFO_RPB (1<<15)
#define B2B_MAX_ALLELES 5
+#define PLP_HAS_SOFT_CLIP(i) ((i)&1)
+#define PLP_SAMPLE_ID(i) ((i)>>1)
+
typedef struct __bcf_callaux_t {
+ int fmt_flag;
int capQ, min_baseQ;
int openQ, extQ, tandemQ; // for indels
uint32_t min_support, max_support; // for collecting indel candidates
void *rghash;
} bcf_callaux_t;
+// per-sample values
typedef struct {
uint32_t ori_depth;
unsigned int mq0;
- int32_t *ADF, *ADR;
+ int32_t *ADF, *ADR, SCR;
float qsum[4];
// The fields are:
// depth fwd .. ref (0) and non-ref (2)
float p[25]; // phred-scaled likelihood of each genotype
} bcf_callret1_t;
+// values for all samples
typedef struct {
int tid, pos;
bcf_hdr_t *bcf_hdr;
int n_supp; // number of supporting non-reference reads
double anno[16];
unsigned int depth, ori_depth, mq0;
- int32_t *PL, *DP4, *ADR, *ADF;
+ int32_t *PL, *DP4, *ADR, *ADF, *SCR;
uint8_t *fmt_arr;
float vdb; // variant distance bias
float mwu_pos, mwu_mq, mwu_bq, mwu_mqs;
#define FT_STDIN (1<<3)
char *bcftools_version(void);
+
+/// Report an error and exit -1
void error(const char *format, ...) HTS_NORETURN HTS_FORMAT(HTS_PRINTF_FMT, 1, 2);
+
+/// Report an error and exit -1. If errno != 0, appends strerror(errno).
+// Note: unlike error() above, the message should not end with "\n" as a
+// newline will be added by the function.
+void error_errno(const char *format, ...) HTS_NORETURN HTS_FORMAT(HTS_PRINTF_FMT, 1, 2);
+
void bcf_hdr_append_version(bcf_hdr_t *hdr, int argc, char **argv, const char *cmd);
const char *hts_bcf_wmode(int file_type);
}
family_t;
+// For the single-sample and grouped -G calling
+typedef struct
+{
+ float *qsum; // QS(quality sum) values
+ int nqsum, dp;
+ double fa,fb,fc,fa2,fb2,fc2,fab,fac,fbc;
+}
+grp1_t;
+typedef struct
+{
+ grp1_t *grp;
+ int ngrp;
+ int *smpl2grp;
+}
+grp_t;
+
+// For the `-C alleles -i` constrained calling
+typedef struct
+{
+ uint32_t n:31, used:1;
+ char **allele;
+}
+tgt_als_t;
+
typedef struct _ccall_t ccall_t;
typedef struct
{
// mcall only
- float *qsum; // QS(sum) values
- int nqsum, npdg;
+ int npdg;
int *als_map, nals_map; // mapping from full set of alleles to trimmed set of alleles (old -> new)
int *pl_map, npl_map; // same as above for PLs, but reverse (new -> old)
char **als; // array to hold the trimmed set of alleles to appear on output
uint16_t *trio[5][5]; // family type, second index: allele count (2-4, first two are unused)
double *GLs;
float *GPs; // FORMAT/GP: posterior probabilities
- int32_t *GQs; // FORMAT/GQ: genotype qualities
+ int32_t *GQs, *ADs; // FORMAT/GQ: genotype qualities; AD: allelic depth for -G
int32_t *itmp; // temporary int array, used for new PLs with CALL_CONSTR_ALLELES
- int n_itmp, nGPs;
+ int n_itmp, nGPs, nADs;
vcmp_t *vcmp;
double trio_Pm_SNPs, trio_Pm_del, trio_Pm_ins; // P(mendelian) for trio calling, see mcall_call_trio_genotypes()
int32_t *ugts, *cgts; // unconstraind and constrained GTs
uint32_t output_tags;
char *prior_AN, *prior_AC; // reference panel AF tags (AF=AC/AN)
+ tgt_als_t *tgt_als; // for CALL_CONSTR_ALLELES
+ char *sample_groups; // for single-sample or grouped calling with -G
+ grp_t smpl_grp;
+ float *qsum;
+ int nqsum;
// ccall only
double indel_frac, min_perm_p, min_lrt;
--- /dev/null
+/*
+ Copyright (C) 2019 Genome Research Ltd.
+
+ Author: Petr Danecek <pd3@sanger.ac.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ DEALINGS IN THE SOFTWARE.
+*/
+
+#include <string.h>
+#include "cols.h"
+
+cols_t *cols_split(const char *line, cols_t *cols, char delim)
+{
+ if ( !cols ) cols = (cols_t*) calloc(1,sizeof(cols_t));
+ if ( cols->rmme ) free(cols->rmme);
+ cols->n = 0;
+ cols->rmme = strdup(line);
+ char *ss = cols->rmme;
+ while (1)
+ {
+ char *se = ss;
+ while ( *se && *se!=delim ) se++;
+ char tmp = *se;
+ *se = 0;
+ cols->n++;
+ if ( cols->n > cols->m )
+ {
+ cols->m += 10;
+ cols->off = (char**) realloc(cols->off, sizeof(*cols->off)*cols->m);
+ }
+ cols->off[ cols->n - 1 ] = ss;
+ if ( !tmp ) break;
+ ss = se + 1;
+ }
+ return cols;
+}
+
+void cols_append(cols_t *cols, char *str)
+{
+ if ( cols->rmme )
+ {
+ size_t str_len = strlen(str);
+ size_t lst_len = strlen(cols->off[ cols->n - 1 ]);
+ size_t tot_len = 2 + str_len + lst_len + (cols->off[ cols->n - 1 ] - cols->rmme);
+
+ cols_t *tmp_cols = (cols_t*)calloc(1,sizeof(cols_t));
+ tmp_cols->rmme = (char*) calloc(tot_len,1);
+ tmp_cols->off = (char**) calloc(cols->n+1,sizeof(*tmp_cols->off));
+
+ char *ptr = tmp_cols->rmme;
+ int i;
+ for (i=0; i<cols->n; i++)
+ {
+ size_t len = strlen(cols->off[i]);
+ memcpy(ptr, cols->off[i], len);
+ tmp_cols->off[i] = ptr;
+ ptr += len + 1;
+ }
+ memcpy(ptr, str, str_len);
+ tmp_cols->off[i] = ptr;
+
+ free(cols->off);
+ free(cols->rmme);
+ cols->rmme = tmp_cols->rmme;
+ cols->off = tmp_cols->off;
+ cols->n = cols->n+1;
+ cols->m = cols->n;
+ free(tmp_cols);
+ return;
+ }
+ cols->n++;
+ if ( cols->n > cols->m )
+ {
+ cols->m++;
+ cols->off = (char**) realloc(cols->off,sizeof(*cols->off)*cols->m);
+ }
+ cols->off[cols->n-1] = str;
+}
+void cols_clear(cols_t *cols)
+{
+ if ( !cols ) return;
+ free(cols->rmme);
+ free(cols->off);
+ cols->rmme = NULL;
+ cols->off = NULL;
+}
+void cols_destroy(cols_t *cols)
+{
+ if ( !cols ) return;
+ cols_clear(cols);
+ free(cols);
+}
+
--- /dev/null
+#include "bcftools.pysam.h"
+
+/*
+ Copyright (C) 2019 Genome Research Ltd.
+
+ Author: Petr Danecek <pd3@sanger.ac.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ DEALINGS IN THE SOFTWARE.
+*/
+
+#include <string.h>
+#include "cols.h"
+
+cols_t *cols_split(const char *line, cols_t *cols, char delim)
+{
+ if ( !cols ) cols = (cols_t*) calloc(1,sizeof(cols_t));
+ if ( cols->rmme ) free(cols->rmme);
+ cols->n = 0;
+ cols->rmme = strdup(line);
+ char *ss = cols->rmme;
+ while (1)
+ {
+ char *se = ss;
+ while ( *se && *se!=delim ) se++;
+ char tmp = *se;
+ *se = 0;
+ cols->n++;
+ if ( cols->n > cols->m )
+ {
+ cols->m += 10;
+ cols->off = (char**) realloc(cols->off, sizeof(*cols->off)*cols->m);
+ }
+ cols->off[ cols->n - 1 ] = ss;
+ if ( !tmp ) break;
+ ss = se + 1;
+ }
+ return cols;
+}
+
+void cols_append(cols_t *cols, char *str)
+{
+ if ( cols->rmme )
+ {
+ size_t str_len = strlen(str);
+ size_t lst_len = strlen(cols->off[ cols->n - 1 ]);
+ size_t tot_len = 2 + str_len + lst_len + (cols->off[ cols->n - 1 ] - cols->rmme);
+
+ cols_t *tmp_cols = (cols_t*)calloc(1,sizeof(cols_t));
+ tmp_cols->rmme = (char*) calloc(tot_len,1);
+ tmp_cols->off = (char**) calloc(cols->n+1,sizeof(*tmp_cols->off));
+
+ char *ptr = tmp_cols->rmme;
+ int i;
+ for (i=0; i<cols->n; i++)
+ {
+ size_t len = strlen(cols->off[i]);
+ memcpy(ptr, cols->off[i], len);
+ tmp_cols->off[i] = ptr;
+ ptr += len + 1;
+ }
+ memcpy(ptr, str, str_len);
+ tmp_cols->off[i] = ptr;
+
+ free(cols->off);
+ free(cols->rmme);
+ cols->rmme = tmp_cols->rmme;
+ cols->off = tmp_cols->off;
+ cols->n = cols->n+1;
+ cols->m = cols->n;
+ free(tmp_cols);
+ return;
+ }
+ cols->n++;
+ if ( cols->n > cols->m )
+ {
+ cols->m++;
+ cols->off = (char**) realloc(cols->off,sizeof(*cols->off)*cols->m);
+ }
+ cols->off[cols->n-1] = str;
+}
+void cols_clear(cols_t *cols)
+{
+ if ( !cols ) return;
+ free(cols->rmme);
+ free(cols->off);
+ cols->rmme = NULL;
+ cols->off = NULL;
+}
+void cols_destroy(cols_t *cols)
+{
+ if ( !cols ) return;
+ cols_clear(cols);
+ free(cols);
+}
+
--- /dev/null
+/*
+ Copyright (C) 2019 Genome Research Ltd.
+
+ Author: Petr Danecek <pd3@sanger.ac.uk>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ DEALINGS IN THE SOFTWARE.
+*/
+
+#ifndef __COLS_H__
+#define __COLS_H__
+
+#include <stdlib.h>
+
+typedef struct
+{
+ int n,m;
+ char **off, *rmme;
+}
+cols_t;
+
+/*
+ cols_split() can be called repeatedly to split new strings, memory is allocated
+ and deallocated automatically
+*/
+cols_t *cols_split(const char *line, cols_t *cols, char delim);
+
+/*
+ Although cols_append() can be combined with cols_split(), it is much slower and
+ the string must exist throughout the life of cols unless initialized with cols_split().
+*/
+void cols_append(cols_t *cols, char *str);
+void cols_clear(cols_t *cols);
+void cols_destroy(cols_t *cols);
+
+#endif
#define PICK_ALT 2
#define PICK_LONG 4
#define PICK_SHORT 8
+#define PICK_IUPAC 16
typedef struct
{
int fa_src_pos; // last genomic coordinate read from the input fasta (0-based)
char prev_base; // this is only to validate the REF allele in the VCF - the modified fa_buf cannot be used for inserts following deletions, see 600#issuecomment-383186778
int prev_base_pos; // the position of prev_base
+ int prev_is_insert;
rbuf_t vcf_rbuf;
bcf1_t **vcf_buf;
int nvcf_buf, rid;
- char *chr;
+ char *chr, *chr_prefix;
regidx_t *mask;
regitr_t *itr;
FILE *fp_out;
FILE *fp_chain;
char **argv;
- int argc, output_iupac, haplotype, allele, isample;
+ int argc, output_iupac, haplotype, allele, isample, napplied;
char *fname, *ref_fname, *sample, *output_fname, *mask_fname, *chain_fname, missing_allele;
}
args_t;
{
args->files = bcf_sr_init();
args->files->require_index = 1;
- if ( !bcf_sr_add_reader(args->files,args->fname) ) error("Failed to open %s: %s\n", args->fname, bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files,args->fname) ) error("Failed to read from %s: %s\n", !strcmp("-",args->fname)?"standard input":args->fname, bcf_sr_strerror(args->files->errnum));
args->hdr = args->files->readers[0].header;
args->isample = -1;
if ( args->sample )
args->vcf_rbuf.n = 0;
bcf_sr_seek(args->files,line,args->fa_ori_pos);
if ( tmp_ptr ) *tmp_ptr = tmp;
- fprintf(args->fp_out,">%s\n",line);
+ fprintf(args->fp_out,">%s%s\n",args->chr_prefix?args->chr_prefix:"",line);
if (args->chain_fname )
{
args->chain = init_chain(args->chain, args->fa_ori_pos);
{
bcf1_t *rec = *rec_ptr;
if ( args->vcf_rbuf.n >= args->vcf_rbuf.m )
- error("FIXME: too many overlapping records near %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ error("FIXME: too many overlapping records near %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
// Insert the new record in the buffer. The line would be overwritten in
// the next bcf_sr_next_line call, therefore we need to swap it with an
if ( !fmt ) return;
if ( fmt->type!=BCF_BT_INT8 )
- error("Todo: GT field represented with BCF_BT_INT8, too many alleles at %s:%d?\n",bcf_seqname(args->hdr,rec),rec->pos+1);
+ error("Todo: GT field represented with BCF_BT_INT8, too many alleles at %s:%"PRId64"?\n",bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
uint8_t *ptr = fmt->p + fmt->size*args->isample;
- if ( args->haplotype )
+
+ enum { use_hap, use_iupac, pick_one } action = use_hap;
+ if ( args->allele==PICK_IUPAC )
+ {
+ if ( !bcf_gt_is_phased(ptr[0]) && !bcf_gt_is_phased(ptr[fmt->n-1]) ) action = use_iupac;
+ }
+ else if ( args->output_iupac ) action = use_iupac;
+ else if ( !args->haplotype ) action = pick_one;
+
+ if ( action==use_hap )
{
if ( args->haplotype > fmt->n )
{
{
if ( !warned_haplotype )
{
- fprintf(stderr, "Can't apply %d-th haplotype at %s:%d. (This warning is printed only once.)\n", args->haplotype,bcf_seqname(args->hdr,rec),rec->pos+1);
+ fprintf(stderr, "Can't apply %d-th haplotype at %s:%"PRId64". (This warning is printed only once.)\n", args->haplotype,bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
warned_haplotype = 1;
}
return;
ialt = bcf_gt_allele(ialt);
}
}
- else if ( args->output_iupac )
+ else if ( action==use_iupac )
{
ialt = ptr[0];
if ( bcf_gt_is_missing(ialt) || ialt==bcf_int32_vector_end )
if ( ialt>=0 )
{
- if ( rec->n_allele <= ialt || rec->n_allele <= jalt ) error("Invalid VCF, too few ALT alleles at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ if ( rec->n_allele <= ialt || rec->n_allele <= jalt ) error("Invalid VCF, too few ALT alleles at %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
if ( ialt!=jalt && !rec->d.allele[ialt][1] && !rec->d.allele[jalt][1] ) // is this a het snp?
{
char ial = rec->d.allele[ialt][0];
{
if ( ptr[i]==(uint8_t)bcf_int8_vector_end ) break;
jalt = bcf_gt_allele(ptr[i]);
- if ( rec->n_allele <= jalt ) error("Broken VCF, too few alts at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ if ( rec->n_allele <= jalt ) error("Broken VCF, too few alts at %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
if ( args->allele & (PICK_LONG|PICK_SHORT) )
{
int len = jalt==0 ? rec->rlen : strlen(rec->d.allele[jalt]);
}
}
if ( !ialt ) return; // ref allele
- if ( rec->n_allele <= ialt ) error("Broken VCF, too few alts at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ if ( rec->n_allele <= ialt ) error("Broken VCF, too few alts at %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
}
else if ( args->output_iupac && !rec->d.allele[0][1] && !rec->d.allele[1][1] )
{
ialt = 1;
}
- // Overlapping variant? Can be still OK iff this is an insertion
- if ( rec->pos <= args->fa_frz_pos && (rec->pos!=args->fa_frz_pos || rec->d.allele[0][0]!=rec->d.allele[ialt][0]) )
+ // Overlapping variant?
+ if ( rec->pos <= args->fa_frz_pos )
{
- fprintf(stderr,"The site %s:%d overlaps with another variant, skipping...\n", bcf_seqname(args->hdr,rec),rec->pos+1);
- return;
+ // Can be still OK iff this is an insertion (and which does not follow another insertion, see #888).
+ // This still may not be enough for more complicated cases with multiple duplicate positions
+ // and other types in between. In such case let the user normalize the VCF and remove duplicates.
+ int overlap = 0;
+ if ( rec->pos < args->fa_frz_pos || !(bcf_get_variant_type(rec,ialt) & VCF_INDEL) ) overlap = 1;
+ else if ( rec->d.var[ialt].n <= 0 || args->prev_is_insert ) overlap = 1;
+
+ if ( overlap )
+ {
+ fprintf(stderr,"The site %s:%"PRId64" overlaps with another variant, skipping...\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
+ return;
+ }
+
}
int len_diff = 0, alen = 0;
int idx = rec->pos - args->fa_ori_pos + args->fa_mod_off;
if ( idx<0 )
{
- fprintf(stderr,"Warning: ignoring overlapping variant starting at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ fprintf(stderr,"Warning: ignoring overlapping variant starting at %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
return;
}
if ( rec->rlen > args->fa_buf.l - idx )
if ( alen > rec->rlen )
{
rec->d.allele[ialt][rec->rlen] = 0;
- fprintf(stderr,"Warning: trimming variant starting at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ fprintf(stderr,"Warning: trimming variant starting at %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
}
}
if ( idx>=args->fa_buf.l )
- error("FIXME: %s:%d .. idx=%d, ori_pos=%d, len=%"PRIu64", off=%d\n",bcf_seqname(args->hdr,rec),rec->pos+1,idx,args->fa_ori_pos,(uint64_t)args->fa_buf.l,args->fa_mod_off);
+ error("FIXME: %s:%"PRId64" .. idx=%d, ori_pos=%d, len=%"PRIu64", off=%d\n",bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1,idx,args->fa_ori_pos,(uint64_t)args->fa_buf.l,args->fa_mod_off);
// sanity check the reference base
if ( rec->d.allele[ialt][0]=='<' )
{
if ( strcasecmp(rec->d.allele[ialt], "<DEL>") )
- error("Symbolic alleles other than <DEL> are currently not supported: %s at %s:%d\n",rec->d.allele[ialt],bcf_seqname(args->hdr,rec),rec->pos+1);
+ error("Symbolic alleles other than <DEL> are currently not supported: %s at %s:%"PRId64"\n",rec->d.allele[ialt],bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
assert( rec->d.allele[0][1]==0 ); // todo: for now expecting strlen(REF) = 1
len_diff = 1-rec->rlen;
rec->d.allele[ialt] = rec->d.allele[0]; // according to VCF spec, REF must precede the event
}
else if ( strncasecmp(rec->d.allele[0],args->fa_buf.s+idx,rec->rlen) )
{
- // This is hacky, handle a special case: if insert follows a deletion (AAC>A, C>CAA),
+ // This is hacky, handle a special case: if SNP or an insert follows a deletion (AAC>A, C>CAA),
// the reference base in fa_buf is lost and the check fails. We do not keep a buffer
// with the original sequence as it should not be necessary, we should encounter max
// one base overlap
args->fa_buf.s[idx+rec->rlen] = 0;
}
error(
- "The fasta sequence does not match the REF allele at %s:%d:\n"
- " .vcf: [%s]\n"
+ "The fasta sequence does not match the REF allele at %s:%"PRId64":\n"
+ " .vcf: [%s] <- (REF)\n"
" .vcf: [%s] <- (ALT)\n"
" .fa: [%s]%c%s\n",
- bcf_seqname(args->hdr,rec),rec->pos+1, rec->d.allele[0], rec->d.allele[ialt], args->fa_buf.s+idx,
+ bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1, rec->d.allele[0], rec->d.allele[ialt], args->fa_buf.s+idx,
tmp?tmp:' ',tmp?args->fa_buf.s+idx+rec->rlen+1:""
);
}
// deletion or same size event
for (i=0; i<alen; i++)
args->fa_buf.s[idx+i] = rec->d.allele[ialt][i];
+
if ( len_diff )
- {
- args->prev_base = rec->d.allele[0][rec->rlen - 1];
- args->prev_base_pos = rec->pos + rec->rlen - 1;
memmove(args->fa_buf.s+idx+alen,args->fa_buf.s+idx+rec->rlen,args->fa_buf.l-idx-rec->rlen);
- }
+
+ args->prev_base = rec->d.allele[0][rec->rlen - 1];
+ args->prev_base_pos = rec->pos + rec->rlen - 1;
+ args->prev_is_insert = 0;
}
else
{
+ args->prev_is_insert = 1;
+ args->prev_base_pos = rec->pos;
+
// insertion
ks_resize(&args->fa_buf, args->fa_buf.l + len_diff);
memmove(args->fa_buf.s + idx + rec->rlen + len_diff, args->fa_buf.s + idx + rec->rlen, args->fa_buf.l - idx - rec->rlen);
- for (i=0; i<alen; i++)
+
+ // This can get tricky, make sure the bases unchanged by the insertion do not overwrite preceeding variants.
+ // For example, here we want to get TAA:
+ // POS REF ALT
+ // 1 C T
+ // 1 C CAA
+ int ibeg = 0;
+ while ( ibeg<alen && rec->d.allele[0][ibeg]==rec->d.allele[ialt][ibeg] && rec->pos + ibeg <= args->prev_base_pos ) ibeg++;
+ for (i=ibeg; i<alen; i++)
args->fa_buf.s[idx+i] = rec->d.allele[ialt][i];
}
if (args->chain && len_diff != 0)
args->fa_buf.l += len_diff;
args->fa_mod_off += len_diff;
args->fa_frz_pos = rec->pos + rec->rlen - 1;
+ args->napplied++;
}
flush_fa_buffer(args, 0);
bgzf_close(fasta);
free(str.s);
+ fprintf(stderr,"Applied %d variants\n", args->napplied);
}
static void usage(args_t *args)
fprintf(stderr, " -f, --fasta-ref <file> reference sequence in fasta format\n");
fprintf(stderr, " -H, --haplotype <which> choose which allele to use from the FORMAT/GT field, note\n");
fprintf(stderr, " the codes are case-insensitive:\n");
- fprintf(stderr, " 1: first allele from GT\n");
- fprintf(stderr, " 2: second allele\n");
+ fprintf(stderr, " 1: first allele from GT, regardless of phasing\n");
+ fprintf(stderr, " 2: second allele from GT, regardless of phasing\n");
fprintf(stderr, " R: REF allele in het genotypes\n");
fprintf(stderr, " A: ALT allele\n");
fprintf(stderr, " LR,LA: longer allele and REF/ALT if equal length\n");
fprintf(stderr, " SR,SA: shorter allele and REF/ALT if equal length\n");
+ fprintf(stderr, " 1pIu,2pIu: first/second allele for phased and IUPAC code for unphased GTs\n");
fprintf(stderr, " -i, --include <expr> select sites for which the expression is true (see man page for details)\n");
fprintf(stderr, " -I, --iupac-codes output variants in the form of IUPAC ambiguity codes\n");
fprintf(stderr, " -m, --mask <file> replace regions with N\n");
fprintf(stderr, " -M, --missing <char> output <char> instead of skipping the missing genotypes\n");
fprintf(stderr, " -o, --output <file> write output to a file [standard output]\n");
+ fprintf(stderr, " -p, --prefix <string> prefix to add to output sequence names\n");
fprintf(stderr, " -s, --sample <name> apply variants of the given sample\n");
fprintf(stderr, "Examples:\n");
fprintf(stderr, " # Get the consensus for one region. The fasta header lines are then expected\n");
{"mask",1,0,'m'},
{"missing",1,0,'M'},
{"chain",1,0,'c'},
+ {"prefix",required_argument,0,'p'},
{0,0,0,0}
};
int c;
- while ((c = getopt_long(argc, argv, "h?s:1Ii:e:H:f:o:m:c:M:",loptions,NULL)) >= 0)
+ while ((c = getopt_long(argc, argv, "h?s:1Ii:e:H:f:o:m:c:M:p:",loptions,NULL)) >= 0)
{
switch (c)
{
+ case 'p': args->chr_prefix = optarg; break;
case 's': args->sample = optarg; break;
case 'o': args->output_fname = optarg; break;
case 'I': args->output_iupac = 1; break;
else if ( !strcasecmp(optarg,"LA") ) args->allele |= PICK_LONG|PICK_ALT;
else if ( !strcasecmp(optarg,"SR") ) args->allele |= PICK_SHORT|PICK_REF;
else if ( !strcasecmp(optarg,"SA") ) args->allele |= PICK_SHORT|PICK_ALT;
+ else if ( !strcasecmp(optarg,"1pIu") ) args->allele |= PICK_IUPAC, args->haplotype = 1;
+ else if ( !strcasecmp(optarg,"2pIu") ) args->allele |= PICK_IUPAC, args->haplotype = 2;
else
{
- args->haplotype = optarg[0] - '0';
- if ( args->haplotype <=0 ) error("Expected positive integer with --haplotype\n");
+ char *tmp;
+ args->haplotype = strtol(optarg, &tmp, 10);
+ if ( tmp==optarg || *tmp ) error("Error: Could not parse --haplotype %s, expected numeric argument\n", optarg);
+ if ( args->haplotype <=0 ) error("Error: Expected positive integer with --haplotype\n");
}
break;
default: usage(args); break;
#define PICK_ALT 2
#define PICK_LONG 4
#define PICK_SHORT 8
+#define PICK_IUPAC 16
typedef struct
{
int fa_src_pos; // last genomic coordinate read from the input fasta (0-based)
char prev_base; // this is only to validate the REF allele in the VCF - the modified fa_buf cannot be used for inserts following deletions, see 600#issuecomment-383186778
int prev_base_pos; // the position of prev_base
+ int prev_is_insert;
rbuf_t vcf_rbuf;
bcf1_t **vcf_buf;
int nvcf_buf, rid;
- char *chr;
+ char *chr, *chr_prefix;
regidx_t *mask;
regitr_t *itr;
FILE *fp_out;
FILE *fp_chain;
char **argv;
- int argc, output_iupac, haplotype, allele, isample;
+ int argc, output_iupac, haplotype, allele, isample, napplied;
char *fname, *ref_fname, *sample, *output_fname, *mask_fname, *chain_fname, missing_allele;
}
args_t;
{
args->files = bcf_sr_init();
args->files->require_index = 1;
- if ( !bcf_sr_add_reader(args->files,args->fname) ) error("Failed to open %s: %s\n", args->fname, bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files,args->fname) ) error("Failed to read from %s: %s\n", !strcmp("-",args->fname)?"standard input":args->fname, bcf_sr_strerror(args->files->errnum));
args->hdr = args->files->readers[0].header;
args->isample = -1;
if ( args->sample )
args->vcf_rbuf.n = 0;
bcf_sr_seek(args->files,line,args->fa_ori_pos);
if ( tmp_ptr ) *tmp_ptr = tmp;
- fprintf(args->fp_out,">%s\n",line);
+ fprintf(args->fp_out,">%s%s\n",args->chr_prefix?args->chr_prefix:"",line);
if (args->chain_fname )
{
args->chain = init_chain(args->chain, args->fa_ori_pos);
{
bcf1_t *rec = *rec_ptr;
if ( args->vcf_rbuf.n >= args->vcf_rbuf.m )
- error("FIXME: too many overlapping records near %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ error("FIXME: too many overlapping records near %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
// Insert the new record in the buffer. The line would be overwritten in
// the next bcf_sr_next_line call, therefore we need to swap it with an
if ( !fmt ) return;
if ( fmt->type!=BCF_BT_INT8 )
- error("Todo: GT field represented with BCF_BT_INT8, too many alleles at %s:%d?\n",bcf_seqname(args->hdr,rec),rec->pos+1);
+ error("Todo: GT field represented with BCF_BT_INT8, too many alleles at %s:%"PRId64"?\n",bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
uint8_t *ptr = fmt->p + fmt->size*args->isample;
- if ( args->haplotype )
+
+ enum { use_hap, use_iupac, pick_one } action = use_hap;
+ if ( args->allele==PICK_IUPAC )
+ {
+ if ( !bcf_gt_is_phased(ptr[0]) && !bcf_gt_is_phased(ptr[fmt->n-1]) ) action = use_iupac;
+ }
+ else if ( args->output_iupac ) action = use_iupac;
+ else if ( !args->haplotype ) action = pick_one;
+
+ if ( action==use_hap )
{
if ( args->haplotype > fmt->n )
{
{
if ( !warned_haplotype )
{
- fprintf(bcftools_stderr, "Can't apply %d-th haplotype at %s:%d. (This warning is printed only once.)\n", args->haplotype,bcf_seqname(args->hdr,rec),rec->pos+1);
+ fprintf(bcftools_stderr, "Can't apply %d-th haplotype at %s:%"PRId64". (This warning is printed only once.)\n", args->haplotype,bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
warned_haplotype = 1;
}
return;
ialt = bcf_gt_allele(ialt);
}
}
- else if ( args->output_iupac )
+ else if ( action==use_iupac )
{
ialt = ptr[0];
if ( bcf_gt_is_missing(ialt) || ialt==bcf_int32_vector_end )
if ( ialt>=0 )
{
- if ( rec->n_allele <= ialt || rec->n_allele <= jalt ) error("Invalid VCF, too few ALT alleles at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ if ( rec->n_allele <= ialt || rec->n_allele <= jalt ) error("Invalid VCF, too few ALT alleles at %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
if ( ialt!=jalt && !rec->d.allele[ialt][1] && !rec->d.allele[jalt][1] ) // is this a het snp?
{
char ial = rec->d.allele[ialt][0];
{
if ( ptr[i]==(uint8_t)bcf_int8_vector_end ) break;
jalt = bcf_gt_allele(ptr[i]);
- if ( rec->n_allele <= jalt ) error("Broken VCF, too few alts at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ if ( rec->n_allele <= jalt ) error("Broken VCF, too few alts at %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
if ( args->allele & (PICK_LONG|PICK_SHORT) )
{
int len = jalt==0 ? rec->rlen : strlen(rec->d.allele[jalt]);
}
}
if ( !ialt ) return; // ref allele
- if ( rec->n_allele <= ialt ) error("Broken VCF, too few alts at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ if ( rec->n_allele <= ialt ) error("Broken VCF, too few alts at %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
}
else if ( args->output_iupac && !rec->d.allele[0][1] && !rec->d.allele[1][1] )
{
ialt = 1;
}
- // Overlapping variant? Can be still OK iff this is an insertion
- if ( rec->pos <= args->fa_frz_pos && (rec->pos!=args->fa_frz_pos || rec->d.allele[0][0]!=rec->d.allele[ialt][0]) )
+ // Overlapping variant?
+ if ( rec->pos <= args->fa_frz_pos )
{
- fprintf(bcftools_stderr,"The site %s:%d overlaps with another variant, skipping...\n", bcf_seqname(args->hdr,rec),rec->pos+1);
- return;
+ // Can be still OK iff this is an insertion (and which does not follow another insertion, see #888).
+ // This still may not be enough for more complicated cases with multiple duplicate positions
+ // and other types in between. In such case let the user normalize the VCF and remove duplicates.
+ int overlap = 0;
+ if ( rec->pos < args->fa_frz_pos || !(bcf_get_variant_type(rec,ialt) & VCF_INDEL) ) overlap = 1;
+ else if ( rec->d.var[ialt].n <= 0 || args->prev_is_insert ) overlap = 1;
+
+ if ( overlap )
+ {
+ fprintf(bcftools_stderr,"The site %s:%"PRId64" overlaps with another variant, skipping...\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
+ return;
+ }
+
}
int len_diff = 0, alen = 0;
int idx = rec->pos - args->fa_ori_pos + args->fa_mod_off;
if ( idx<0 )
{
- fprintf(bcftools_stderr,"Warning: ignoring overlapping variant starting at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ fprintf(bcftools_stderr,"Warning: ignoring overlapping variant starting at %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
return;
}
if ( rec->rlen > args->fa_buf.l - idx )
if ( alen > rec->rlen )
{
rec->d.allele[ialt][rec->rlen] = 0;
- fprintf(bcftools_stderr,"Warning: trimming variant starting at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
+ fprintf(bcftools_stderr,"Warning: trimming variant starting at %s:%"PRId64"\n", bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
}
}
if ( idx>=args->fa_buf.l )
- error("FIXME: %s:%d .. idx=%d, ori_pos=%d, len=%"PRIu64", off=%d\n",bcf_seqname(args->hdr,rec),rec->pos+1,idx,args->fa_ori_pos,(uint64_t)args->fa_buf.l,args->fa_mod_off);
+ error("FIXME: %s:%"PRId64" .. idx=%d, ori_pos=%d, len=%"PRIu64", off=%d\n",bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1,idx,args->fa_ori_pos,(uint64_t)args->fa_buf.l,args->fa_mod_off);
// sanity check the reference base
if ( rec->d.allele[ialt][0]=='<' )
{
if ( strcasecmp(rec->d.allele[ialt], "<DEL>") )
- error("Symbolic alleles other than <DEL> are currently not supported: %s at %s:%d\n",rec->d.allele[ialt],bcf_seqname(args->hdr,rec),rec->pos+1);
+ error("Symbolic alleles other than <DEL> are currently not supported: %s at %s:%"PRId64"\n",rec->d.allele[ialt],bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
assert( rec->d.allele[0][1]==0 ); // todo: for now expecting strlen(REF) = 1
len_diff = 1-rec->rlen;
rec->d.allele[ialt] = rec->d.allele[0]; // according to VCF spec, REF must precede the event
}
else if ( strncasecmp(rec->d.allele[0],args->fa_buf.s+idx,rec->rlen) )
{
- // This is hacky, handle a special case: if insert follows a deletion (AAC>A, C>CAA),
+ // This is hacky, handle a special case: if SNP or an insert follows a deletion (AAC>A, C>CAA),
// the reference base in fa_buf is lost and the check fails. We do not keep a buffer
// with the original sequence as it should not be necessary, we should encounter max
// one base overlap
args->fa_buf.s[idx+rec->rlen] = 0;
}
error(
- "The fasta sequence does not match the REF allele at %s:%d:\n"
- " .vcf: [%s]\n"
+ "The fasta sequence does not match the REF allele at %s:%"PRId64":\n"
+ " .vcf: [%s] <- (REF)\n"
" .vcf: [%s] <- (ALT)\n"
" .fa: [%s]%c%s\n",
- bcf_seqname(args->hdr,rec),rec->pos+1, rec->d.allele[0], rec->d.allele[ialt], args->fa_buf.s+idx,
+ bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1, rec->d.allele[0], rec->d.allele[ialt], args->fa_buf.s+idx,
tmp?tmp:' ',tmp?args->fa_buf.s+idx+rec->rlen+1:""
);
}
// deletion or same size event
for (i=0; i<alen; i++)
args->fa_buf.s[idx+i] = rec->d.allele[ialt][i];
+
if ( len_diff )
- {
- args->prev_base = rec->d.allele[0][rec->rlen - 1];
- args->prev_base_pos = rec->pos + rec->rlen - 1;
memmove(args->fa_buf.s+idx+alen,args->fa_buf.s+idx+rec->rlen,args->fa_buf.l-idx-rec->rlen);
- }
+
+ args->prev_base = rec->d.allele[0][rec->rlen - 1];
+ args->prev_base_pos = rec->pos + rec->rlen - 1;
+ args->prev_is_insert = 0;
}
else
{
+ args->prev_is_insert = 1;
+ args->prev_base_pos = rec->pos;
+
// insertion
ks_resize(&args->fa_buf, args->fa_buf.l + len_diff);
memmove(args->fa_buf.s + idx + rec->rlen + len_diff, args->fa_buf.s + idx + rec->rlen, args->fa_buf.l - idx - rec->rlen);
- for (i=0; i<alen; i++)
+
+ // This can get tricky, make sure the bases unchanged by the insertion do not overwrite preceeding variants.
+ // For example, here we want to get TAA:
+ // POS REF ALT
+ // 1 C T
+ // 1 C CAA
+ int ibeg = 0;
+ while ( ibeg<alen && rec->d.allele[0][ibeg]==rec->d.allele[ialt][ibeg] && rec->pos + ibeg <= args->prev_base_pos ) ibeg++;
+ for (i=ibeg; i<alen; i++)
args->fa_buf.s[idx+i] = rec->d.allele[ialt][i];
}
if (args->chain && len_diff != 0)
args->fa_buf.l += len_diff;
args->fa_mod_off += len_diff;
args->fa_frz_pos = rec->pos + rec->rlen - 1;
+ args->napplied++;
}
flush_fa_buffer(args, 0);
bgzf_close(fasta);
free(str.s);
+ fprintf(bcftools_stderr,"Applied %d variants\n", args->napplied);
}
static void usage(args_t *args)
fprintf(bcftools_stderr, " -f, --fasta-ref <file> reference sequence in fasta format\n");
fprintf(bcftools_stderr, " -H, --haplotype <which> choose which allele to use from the FORMAT/GT field, note\n");
fprintf(bcftools_stderr, " the codes are case-insensitive:\n");
- fprintf(bcftools_stderr, " 1: first allele from GT\n");
- fprintf(bcftools_stderr, " 2: second allele\n");
+ fprintf(bcftools_stderr, " 1: first allele from GT, regardless of phasing\n");
+ fprintf(bcftools_stderr, " 2: second allele from GT, regardless of phasing\n");
fprintf(bcftools_stderr, " R: REF allele in het genotypes\n");
fprintf(bcftools_stderr, " A: ALT allele\n");
fprintf(bcftools_stderr, " LR,LA: longer allele and REF/ALT if equal length\n");
fprintf(bcftools_stderr, " SR,SA: shorter allele and REF/ALT if equal length\n");
+ fprintf(bcftools_stderr, " 1pIu,2pIu: first/second allele for phased and IUPAC code for unphased GTs\n");
fprintf(bcftools_stderr, " -i, --include <expr> select sites for which the expression is true (see man page for details)\n");
fprintf(bcftools_stderr, " -I, --iupac-codes output variants in the form of IUPAC ambiguity codes\n");
fprintf(bcftools_stderr, " -m, --mask <file> replace regions with N\n");
fprintf(bcftools_stderr, " -M, --missing <char> output <char> instead of skipping the missing genotypes\n");
fprintf(bcftools_stderr, " -o, --output <file> write output to a file [standard output]\n");
+ fprintf(bcftools_stderr, " -p, --prefix <string> prefix to add to output sequence names\n");
fprintf(bcftools_stderr, " -s, --sample <name> apply variants of the given sample\n");
fprintf(bcftools_stderr, "Examples:\n");
fprintf(bcftools_stderr, " # Get the consensus for one region. The fasta header lines are then expected\n");
{"mask",1,0,'m'},
{"missing",1,0,'M'},
{"chain",1,0,'c'},
+ {"prefix",required_argument,0,'p'},
{0,0,0,0}
};
int c;
- while ((c = getopt_long(argc, argv, "h?s:1Ii:e:H:f:o:m:c:M:",loptions,NULL)) >= 0)
+ while ((c = getopt_long(argc, argv, "h?s:1Ii:e:H:f:o:m:c:M:p:",loptions,NULL)) >= 0)
{
switch (c)
{
+ case 'p': args->chr_prefix = optarg; break;
case 's': args->sample = optarg; break;
case 'o': args->output_fname = optarg; break;
case 'I': args->output_iupac = 1; break;
else if ( !strcasecmp(optarg,"LA") ) args->allele |= PICK_LONG|PICK_ALT;
else if ( !strcasecmp(optarg,"SR") ) args->allele |= PICK_SHORT|PICK_REF;
else if ( !strcasecmp(optarg,"SA") ) args->allele |= PICK_SHORT|PICK_ALT;
+ else if ( !strcasecmp(optarg,"1pIu") ) args->allele |= PICK_IUPAC, args->haplotype = 1;
+ else if ( !strcasecmp(optarg,"2pIu") ) args->allele |= PICK_IUPAC, args->haplotype = 2;
else
{
- args->haplotype = optarg[0] - '0';
- if ( args->haplotype <=0 ) error("Expected positive integer with --haplotype\n");
+ char *tmp;
+ args->haplotype = strtol(optarg, &tmp, 10);
+ if ( tmp==optarg || *tmp ) error("Error: Could not parse --haplotype %s, expected numeric argument\n", optarg);
+ if ( args->haplotype <=0 ) error("Error: Expected positive integer with --haplotype\n");
}
break;
default: usage(args); break;
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
+#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include <math.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/vcfutils.h>
+#include <htslib/kfunc.h>
#include "bcftools.h"
+#include "variantkey.h"
#include "convert.h"
#define T_CHROM 1
#define T_END 27
#define T_POS0 28
#define T_END0 29
+#define T_RSX 30 // RSID HEX
+#define T_VKX 31 // VARIANTKEY HEX
+#define T_PBINOM 32
typedef struct _fmt_t
{
}
static void process_info(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
{
+ int i;
+ if ( !fmt->key ) // the whole INFO column
+ {
+ int first = 1;
+ for (i=0; i<line->n_info; i++)
+ {
+ bcf_info_t *inf = &line->d.info[i];
+ if ( !inf->vptr ) continue;
+ if ( !first ) kputc(';', str);
+ first = 0;
+ if ( inf->key >= convert->header->n[BCF_DT_ID] ) continue;
+ kputs(convert->header->id[BCF_DT_ID][inf->key].key, str);
+ if ( inf->len <= 0 ) continue;
+ kputc('=', str);
+ if ( inf->len == 1 )
+ {
+ switch (inf->type)
+ {
+ case BCF_BT_INT8: if ( inf->v1.i==bcf_int8_missing ) kputc('.', str); else kputw(inf->v1.i, str); break;
+ case BCF_BT_INT16: if ( inf->v1.i==bcf_int16_missing ) kputc('.', str); else kputw(inf->v1.i, str); break;
+ case BCF_BT_INT32: if ( inf->v1.i==bcf_int32_missing ) kputc('.', str); else kputw(inf->v1.i, str); break;
+ case BCF_BT_FLOAT: if ( bcf_float_is_missing(inf->v1.f) ) kputc('.', str); else kputd(inf->v1.f, str); break;
+ case BCF_BT_CHAR: kputc(inf->v1.i, str); break;
+ default: error("Unexpected type %d", inf->type); break;
+ }
+ }
+ else bcf_fmt_array(str, inf->len, inf->type, inf->vptr);
+ }
+ if ( first ) kputc('.', str);
+ return;
+ }
+
if ( fmt->id<0 )
{
kputc('.', str);
return;
}
- int i;
for (i=0; i<line->n_info; i++)
if ( line->d.info[i].key == fmt->id ) break;
fmt->ready = 1;
}
+static void process_complete_format(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
+{
+ if ( convert->nsamples )
+ {
+ int i,j;
+ if ( line->n_fmt)
+ {
+ int gt_i = -1;
+ bcf_fmt_t *fmt = line->d.fmt;
+ int first = 1;
+ for (i=0; i<(int)line->n_fmt; i++)
+ {
+ if ( !fmt[i].p || fmt[i].id<0 ) continue;
+ if ( !first ) kputc(':', str);
+ first = 0;
+ kputs(convert->header->id[BCF_DT_ID][fmt[i].id].key, str);
+ if ( strcmp(convert->header->id[BCF_DT_ID][fmt[i].id].key, "GT") == 0) gt_i = i;
+ }
+ if ( first ) kputc('.', str);
+ for (j=0; j<convert->nsamples; j++)
+ {
+ kputc('\t', str);
+ first = 1;
+ for (i=0; i<(int)line->n_fmt; i++)
+ {
+ bcf_fmt_t *f = &fmt[i];
+ if ( !f->p ) continue;
+ if ( !first ) kputc(':', str);
+ first = 0;
+ if (gt_i == i)
+ bcf_format_gt(f,convert->samples[j],str);
+ else
+ bcf_fmt_array(str, f->n, f->type, f->p + convert->samples[j] * f->size);
+ }
+ if ( first ) kputc('.', str);
+ }
+ }
+ else
+ for (j=0; j<=line->n_sample; j++)
+ kputs("\t.", str);
+ }
+ else
+ kputc('.',str);
+}
static void process_format(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
{
if ( !fmt->ready )
if ( line_type & VCF_INDEL ) { if (i) kputc(',',str); kputs("INDEL", str); i++; }
if ( line_type & VCF_OTHER ) { if (i) kputc(',',str); kputs("OTHER", str); i++; }
if ( line_type & VCF_BND ) { if (i) kputc(',',str); kputs("BND", str); i++; }
+ if ( line_type & VCF_OVERLAP ) { if (i) kputc(',',str); kputs("OVERLAP", str); i++; }
}
static void process_line(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
{
// for (i=0; i<convert->nsamples; i++) kputs(" 0.33 0.33 0.33", str);
// return;
- error("Error parsing GT tag at %s:%d\n", bcf_seqname(convert->header,line),line->pos+1);
+ error("Error parsing GT tag at %s:%"PRId64"\n", bcf_seqname(convert->header,line),(int64_t) line->pos+1);
}
n /= convert->nsamples;
// for (i=0; i<convert->nsamples; i++) kputs(" 0.33 0.33 0.33", str);
// return;
- error("Error parsing PL tag at %s:%d\n", bcf_seqname(convert->header,line),line->pos+1);
+ error("Error parsing PL tag at %s:%"PRId64"\n", bcf_seqname(convert->header,line),(int64_t) line->pos+1);
}
n /= convert->nsamples;
// for (i=0; i<convert->nsamples; i++) kputs(" 0.33 0.33 0.33", str);
// return;
- error("Error parsing GP tag at %s:%d\n", bcf_seqname(convert->header,line),line->pos+1);
+ error("Error parsing GP tag at %s:%"PRId64"\n", bcf_seqname(convert->header,line),(int64_t) line->pos+1);
}
n /= convert->nsamples;
{
if ( ptr[j]==bcf_int32_vector_end ) break;
if ( ptr[j]==bcf_int32_missing ) { ptr[j]=0; continue; }
- if ( ptr[j]<0 || ptr[j]>1 ) error("[%s:%d:%f] GP value outside range [0,1]; bcftools convert expects the VCF4.3+ spec for the GP field encoding genotype posterior probabilities", bcf_seqname(convert->header,line),line->pos+1,ptr[j]);
+ if ( ptr[j]<0 || ptr[j]>1 ) error("[%s:%"PRId64":%f] GP value outside range [0,1]; bcftools convert expects the VCF4.3+ spec for the GP field encoding genotype posterior probabilities", bcf_seqname(convert->header,line),(int64_t) line->pos+1,ptr[j]);
sum+=ptr[j];
}
if ( j==line->n_allele )
int i, gt_id = bcf_hdr_id2int(convert->header, BCF_DT_ID, "GT");
if ( !bcf_hdr_idinfo_exists(convert->header,BCF_HL_FMT,gt_id) )
- error("FORMAT/GT tag not present at %s:%d\n", bcf_seqname(convert->header, line), line->pos+1);
+ error("FORMAT/GT tag not present at %s:%"PRId64"\n", bcf_seqname(convert->header, line),(int64_t) line->pos+1);
if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);
bcf_fmt_t *fmt_gt = NULL;
for (i=0; i<line->n_fmt; i++)
if ( line->d.fmt[i].id==gt_id ) { fmt_gt = &line->d.fmt[i]; break; }
if ( !fmt_gt )
- error("FORMAT/GT tag not present at %s:%d\n", bcf_seqname(convert->header, line), line->pos+1);
+ error("FORMAT/GT tag not present at %s:%"PRId64"\n", bcf_seqname(convert->header, line),(int64_t) line->pos+1);
// Alloc all memory in advance to avoid kput routines. The biggest allowed allele index is 99
if ( line->n_allele > 100 )
- error("Too many alleles (%d) at %s:%d\n", line->n_allele, bcf_seqname(convert->header, line), line->pos+1);
+ error("Too many alleles (%d) at %s:%"PRId64"\n", line->n_allele, bcf_seqname(convert->header, line),(int64_t) line->pos+1);
if ( ks_resize(str, str->l+convert->nsamples*8) != 0 )
- error("Could not alloc %"PRIu64" bytes\n", (uint64_t)(str->l + convert->nsamples*8));
+ error("Could not alloc %" PRIu64 " bytes\n", (uint64_t)(str->l + convert->nsamples*8));
if ( fmt_gt->type!=BCF_BT_INT8 ) // todo: use BRANCH_INT if the VCF is valid
- error("Uh, too many alleles (%d) or redundant BCF representation at %s:%d\n", line->n_allele, bcf_seqname(convert->header, line), line->pos+1);
+ error("Uh, too many alleles (%d) or redundant BCF representation at %s:%"PRId64"\n", line->n_allele, bcf_seqname(convert->header, line),(int64_t) line->pos+1);
if ( fmt_gt->n!=1 && fmt_gt->n!=2 )
- error("Uh, ploidy of %d not supported, see %s:%d\n", fmt_gt->n, bcf_seqname(convert->header, line), line->pos+1);
+ error("Uh, ploidy of %d not supported, see %s:%"PRId64"\n", fmt_gt->n, bcf_seqname(convert->header, line),(int64_t) line->pos+1);
int8_t *ptr = ((int8_t*) fmt_gt->p) - fmt_gt->n;
for (i=0; i<convert->nsamples; i++)
int i, gt_id = bcf_hdr_id2int(convert->header, BCF_DT_ID, "GT");
if ( !bcf_hdr_idinfo_exists(convert->header,BCF_HL_FMT,gt_id) )
- error("FORMAT/GT tag not present at %s:%d\n", bcf_seqname(convert->header, line), line->pos+1);
+ error("FORMAT/GT tag not present at %s:%"PRId64"\n", bcf_seqname(convert->header, line),(int64_t) line->pos+1);
if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);
bcf_fmt_t *fmt_gt = NULL;
for (i=0; i<line->n_fmt; i++)
if ( line->d.fmt[i].id==gt_id ) { fmt_gt = &line->d.fmt[i]; break; }
if ( !fmt_gt )
- error("FORMAT/GT tag not present at %s:%d\n", bcf_seqname(convert->header, line), line->pos+1);
+ error("FORMAT/GT tag not present at %s:%"PRId64"\n", bcf_seqname(convert->header, line),(int64_t) line->pos+1);
// Alloc all memory in advance to avoid kput routines. The biggest allowed allele index is 99
if ( line->n_allele > 100 )
- error("Too many alleles (%d) at %s:%d\n", line->n_allele, bcf_seqname(convert->header, line), line->pos+1);
+ error("Too many alleles (%d) at %s:%"PRId64"\n", line->n_allele, bcf_seqname(convert->header, line),(int64_t) line->pos+1);
if ( ks_resize(str, str->l+convert->nsamples*8) != 0 )
- error("Could not alloc %"PRIu64" bytes\n", (uint64_t)(str->l + convert->nsamples*8));
+ error("Could not alloc %" PRIu64 " bytes\n", (uint64_t)(str->l + convert->nsamples*8));
if ( fmt_gt->type!=BCF_BT_INT8 ) // todo: use BRANCH_INT if the VCF is valid
- error("Uh, too many alleles (%d) or redundant BCF representation at %s:%d\n", line->n_allele, bcf_seqname(convert->header, line), line->pos+1);
+ error("Uh, too many alleles (%d) or redundant BCF representation at %s:%"PRId64"\n", line->n_allele, bcf_seqname(convert->header, line),(int64_t) line->pos+1);
int8_t *ptr = ((int8_t*) fmt_gt->p) - fmt_gt->n;
for (i=0; i<convert->nsamples; i++)
str->s[--str->l] = 0; // delete the last space
}
+static void process_rsid_hex(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
+{
+ char *ptr = line->d.id;
+ ptr += 2; // remove 'rs'
+ ksprintf(str, "%08" PRIx32 "", (uint32_t)strtoul(ptr, NULL, 10));
+}
+
+static void process_variantkey_hex(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
+{
+ uint64_t vk = variantkey(
+ convert->header->id[BCF_DT_CTG][line->rid].key,
+ strlen(convert->header->id[BCF_DT_CTG][line->rid].key),
+ line->pos,
+ line->d.allele[0],
+ strlen(line->d.allele[0]),
+ line->d.allele[1],
+ strlen(line->d.allele[1]));
+ ksprintf(str, "%016" PRIx64 "", vk);
+}
+
+static void process_pbinom(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
+{
+ int i;
+ if ( !fmt->ready )
+ {
+ fmt->fmt = NULL; // AD
+ fmt->usr = NULL; // GT
+
+ for (i=0; i<(int)line->n_fmt; i++)
+ if ( line->d.fmt[i].id==fmt->id ) { fmt->fmt = &line->d.fmt[i]; break; }
+
+ // Check that the first field is GT
+ int gt_id = bcf_hdr_id2int(convert->header, BCF_DT_ID, "GT");
+ if ( !bcf_hdr_idinfo_exists(convert->header, BCF_HL_FMT, fmt->id) ) error("Error: FORMAT/GT is not defined in the header\n");
+ for (i=0; i<(int)line->n_fmt; i++)
+ if ( line->d.fmt[i].id==gt_id ) { fmt->usr = &line->d.fmt[i]; break; } // it should always be first according to VCF spec, but...
+
+ if ( fmt->usr && line->d.fmt[i].type!=BCF_BT_INT8 ) // skip sites with many alleles
+ fmt->usr = NULL;
+
+ fmt->ready = 1;
+ }
+ bcf_fmt_t *gt_fmt = (bcf_fmt_t*) fmt->usr;
+ if ( !fmt->fmt || !gt_fmt || gt_fmt->n!=2 ) goto invalid;
+
+ int n[2] = {0,0};
+ int8_t *gt = (int8_t*)(gt_fmt->p + isample*gt_fmt->size);
+ for (i=0; i<2; i++)
+ {
+ if ( bcf_gt_is_missing(gt[i]) || gt[i] == bcf_int8_vector_end ) goto invalid;
+ int al = bcf_gt_allele(gt[i]);
+ if ( al > line->n_allele || al >= fmt->fmt->n ) goto invalid;
+
+ #define BRANCH(type_t, missing, vector_end) { \
+ type_t val = ((type_t *) fmt->fmt->p)[al + isample*fmt->fmt->n]; \
+ if ( val==missing || val==vector_end ) goto invalid; \
+ else n[i] = val; \
+ }
+ switch (fmt->fmt->type)
+ {
+ case BCF_BT_INT8: BRANCH(int8_t, bcf_int8_missing, bcf_int8_vector_end); break;
+ case BCF_BT_INT16: BRANCH(int16_t, bcf_int16_missing, bcf_int16_vector_end); break;
+ case BCF_BT_INT32: BRANCH(int32_t, bcf_int32_missing, bcf_int32_vector_end); break;
+ default: goto invalid; break;
+ }
+ #undef BRANCH
+ }
+
+ if ( n[0]==n[1] ) kputc(n[0]==0 ? '.':'0', str);
+ else
+ {
+ double pval = n[0] < n[1] ? kf_betai(n[1], n[0] + 1, 0.5) : kf_betai(n[0], n[1] + 1, 0.5);
+ pval *= 2;
+ assert( pval-1 < 1e-10 );
+ if ( pval>=1 ) pval = 0; // this can happen, machine precision error, eg. kf_betai(1,0,0.5)
+ else
+ pval = -4.34294481903*log(pval);
+ kputd(pval, str);
+ }
+ return;
+
+invalid:
+ kputc('.', str);
+}
+
static fmt_t *register_tag(convert_t *convert, int type, char *key, int is_gtf)
{
convert->nfmt++;
else if ( !strcmp("QUAL",key) ) { fmt->type = T_QUAL; }
else if ( !strcmp("FILTER",key) ) { fmt->type = T_FILTER; }
else if ( !strcmp("_CHROM_POS_ID",key) ) { fmt->type = T_CHROM_POS_ID; }
- else if ( id>=0 && bcf_hdr_idinfo_exists(convert->header,BCF_HL_INFO,id) )
- {
- fmt->type = T_INFO;
- fprintf(stderr,"Warning: Assuming INFO/%s\n", key);
- }
+ else if ( !strcmp("RSX",key) ) { fmt->type = T_RSX; }
+ else if ( !strcmp("VKX",key) ) { fmt->type = T_VKX; }
+ else if ( id>=0 && bcf_hdr_idinfo_exists(convert->header,BCF_HL_INFO,id) ) { fmt->type = T_INFO; }
+ }
+ if ( fmt->type==T_PBINOM )
+ {
+ fmt->id = bcf_hdr_id2int(convert->header, BCF_DT_ID, fmt->key);
+ if ( !bcf_hdr_idinfo_exists(convert->header,BCF_HL_FMT, fmt->id) ) error("No such FORMAT tag defined in the header: %s\n", fmt->key);
}
}
case T_CHROM: fmt->handler = &process_chrom; break;
case T_POS: fmt->handler = &process_pos; break;
case T_POS0: fmt->handler = &process_pos0; break;
- case T_END: fmt->handler = &process_end; break;
- case T_END0: fmt->handler = &process_end0; break;
+ case T_END: fmt->handler = &process_end; convert->max_unpack |= BCF_UN_INFO; break;
+ case T_END0: fmt->handler = &process_end0; convert->max_unpack |= BCF_UN_INFO; break;
case T_ID: fmt->handler = &process_id; break;
case T_REF: fmt->handler = &process_ref; break;
case T_ALT: fmt->handler = &process_alt; break;
case T_QUAL: fmt->handler = &process_qual; break;
case T_FILTER: fmt->handler = &process_filter; convert->max_unpack |= BCF_UN_FLT; break;
case T_INFO: fmt->handler = &process_info; convert->max_unpack |= BCF_UN_INFO; break;
- case T_FORMAT: fmt->handler = &process_format; convert->max_unpack |= BCF_UN_FMT; break;
+ case T_FORMAT: fmt->handler = fmt->key ? &process_format : &process_complete_format; convert->max_unpack |= BCF_UN_FMT; break;
case T_SAMPLE: fmt->handler = &process_sample; break;
case T_SEP: fmt->handler = &process_sep; break;
case T_IS_TS: fmt->handler = &process_is_ts; break;
case T_GT_TO_HAP2: fmt->handler = &process_gt_to_hap2; convert->max_unpack |= BCF_UN_FMT; break;
case T_TBCSQ: fmt->handler = &process_tbcsq; fmt->destroy = &destroy_tbcsq; convert->max_unpack |= BCF_UN_FMT; break;
case T_LINE: fmt->handler = &process_line; convert->max_unpack |= BCF_UN_FMT; break;
+ case T_RSX: fmt->handler = &process_rsid_hex; break;
+ case T_VKX: fmt->handler = &process_variantkey_hex; break;
+ case T_PBINOM: fmt->handler = &process_pbinom; convert->max_unpack |= BCF_UN_FMT; break;
default: error("TODO: handler for type %d\n", fmt->type);
}
if ( key && fmt->type==T_INFO )
else if ( !strcmp(str.s, "IUPACGT") ) register_tag(convert, T_IUPAC_GT, "GT", is_gtf);
else if ( !strcmp(str.s, "INFO") )
{
- if ( *q!='/' ) error("Could not parse format string: %s\n", convert->format_str);
+ if ( *q!='/' )
+ {
+ int id = bcf_hdr_id2int(convert->header, BCF_DT_ID, str.s);
+ if ( bcf_hdr_idinfo_exists(convert->header,BCF_HL_INFO,id) )
+ error("Could not parse format string \"%s\". Did you mean %%INFO/%s?\n", convert->format_str,str.s);
+ else
+ error("Could not parse format string: %s\n", convert->format_str);
+ }
p = ++q;
str.l = 0;
while ( *q && (isalnum(*q) || *q=='_' || *q=='.') ) q++;
fmt_t *fmt = register_tag(convert, T_INFO, str.s, is_gtf);
fmt->subscript = parse_subscript(&q);
}
+ else if ( !strcmp(str.s,"PBINOM") )
+ {
+ if ( *q!='(' ) error("Could not parse the expression: %s\n", convert->format_str);
+ p = ++q;
+ str.l = 0;
+ while ( *q && *q!=')' ) q++;
+ if ( q-p==0 ) error("Could not parse format string: %s\n", convert->format_str);
+ kputsn(p, q-p, &str);
+ register_tag(convert, T_PBINOM, str.s, is_gtf);
+ q++;
+ }
else
{
fmt_t *fmt = register_tag(convert, T_FORMAT, str.s, is_gtf);
else if ( !strcmp(str.s, "_GP_TO_PROB3") ) register_tag(convert, T_GP_TO_PROB3, str.s, is_gtf);
else if ( !strcmp(str.s, "_GT_TO_HAP") ) register_tag(convert, T_GT_TO_HAP, str.s, is_gtf);
else if ( !strcmp(str.s, "_GT_TO_HAP2") ) register_tag(convert, T_GT_TO_HAP2, str.s, is_gtf);
+ else if ( !strcmp(str.s, "RSX") ) register_tag(convert, T_RSX, str.s, is_gtf);
+ else if ( !strcmp(str.s, "VKX") ) register_tag(convert, T_VKX, str.s, is_gtf);
+ else if ( !strcmp(str.s,"pbinom") ) error("Error: pbinom() is currently supported only with FORMAT tags. (todo)\n");
else if ( !strcmp(str.s, "INFO") )
{
- if ( *q!='/' ) error("Could not parse format string: %s\n", convert->format_str);
- p = ++q;
- str.l = 0;
- while ( *q && (isalnum(*q) || *q=='_' || *q=='.') ) q++;
- if ( q-p==0 ) error("Could not parse format string: %s\n", convert->format_str);
- kputsn(p, q-p, &str);
- fmt_t *fmt = register_tag(convert, T_INFO, str.s, is_gtf);
- fmt->subscript = parse_subscript(&q);
+ if ( *q=='/' )
+ {
+ p = ++q;
+ str.l = 0;
+ while ( *q && (isalnum(*q) || *q=='_' || *q=='.') ) q++;
+ if ( q-p==0 ) error("Could not parse format string: %s\n", convert->format_str);
+ kputsn(p, q-p, &str);
+ fmt_t *fmt = register_tag(convert, T_INFO, str.s, is_gtf);
+ fmt->subscript = parse_subscript(&q);
+ }
+ else
+ register_tag(convert, T_INFO, NULL, is_gtf); // the whole INFO
}
+ else if ( !strcmp(str.s, "FORMAT") )
+ register_tag(convert, T_FORMAT, NULL, 0);
else
{
fmt_t *fmt = register_tag(convert, T_INFO, str.s, is_gtf);
int convert_line(convert_t *convert, bcf1_t *line, kstring_t *str)
{
if ( !convert->allow_undef_tags && convert->undef_info_tag )
- error("Error: no such tag defined in the VCF header: INFO/%s. FORMAT fields must be in square brackets, e.g. \"[ %s]\"\n", convert->undef_info_tag,convert->undef_info_tag);
+ {
+ kstring_t msg = {0,0,0};
+ ksprintf(&msg,"Error: no such tag defined in the VCF header: INFO/%s", convert->undef_info_tag);
+
+ int hdr_id = bcf_hdr_id2int(convert->header,BCF_DT_ID,convert->undef_info_tag);
+ if ( hdr_id>=0 && bcf_hdr_idinfo_exists(convert->header,BCF_HL_FMT,hdr_id) )
+ ksprintf(&msg,". FORMAT fields must be enclosed in square brackets, e.g. \"[ %%%s]\"", convert->undef_info_tag);
+ error("%s\n", msg.s);
+ }
int l_ori = str->l;
bcf_unpack(line, convert->max_unpack);
for (js=0; js<convert->nsamples; js++)
{
// Skip samples when filtering was requested
- if ( *convert->subset_samples && !(*convert->subset_samples)[js] ) continue;
+ if ( convert->subset_samples && *convert->subset_samples && !(*convert->subset_samples)[js] ) continue;
// Here comes a hack designed for TBCSQ. When running on large files,
// such as 1000GP, there are too many empty fields in the output and
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
+#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include <math.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/vcfutils.h>
+#include <htslib/kfunc.h>
#include "bcftools.h"
+#include "variantkey.h"
#include "convert.h"
#define T_CHROM 1
#define T_END 27
#define T_POS0 28
#define T_END0 29
+#define T_RSX 30 // RSID HEX
+#define T_VKX 31 // VARIANTKEY HEX
+#define T_PBINOM 32
typedef struct _fmt_t
{
}
static void process_info(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
{
+ int i;
+ if ( !fmt->key ) // the whole INFO column
+ {
+ int first = 1;
+ for (i=0; i<line->n_info; i++)
+ {
+ bcf_info_t *inf = &line->d.info[i];
+ if ( !inf->vptr ) continue;
+ if ( !first ) kputc(';', str);
+ first = 0;
+ if ( inf->key >= convert->header->n[BCF_DT_ID] ) continue;
+ kputs(convert->header->id[BCF_DT_ID][inf->key].key, str);
+ if ( inf->len <= 0 ) continue;
+ kputc('=', str);
+ if ( inf->len == 1 )
+ {
+ switch (inf->type)
+ {
+ case BCF_BT_INT8: if ( inf->v1.i==bcf_int8_missing ) kputc('.', str); else kputw(inf->v1.i, str); break;
+ case BCF_BT_INT16: if ( inf->v1.i==bcf_int16_missing ) kputc('.', str); else kputw(inf->v1.i, str); break;
+ case BCF_BT_INT32: if ( inf->v1.i==bcf_int32_missing ) kputc('.', str); else kputw(inf->v1.i, str); break;
+ case BCF_BT_FLOAT: if ( bcf_float_is_missing(inf->v1.f) ) kputc('.', str); else kputd(inf->v1.f, str); break;
+ case BCF_BT_CHAR: kputc(inf->v1.i, str); break;
+ default: error("Unexpected type %d", inf->type); break;
+ }
+ }
+ else bcf_fmt_array(str, inf->len, inf->type, inf->vptr);
+ }
+ if ( first ) kputc('.', str);
+ return;
+ }
+
if ( fmt->id<0 )
{
kputc('.', str);
return;
}
- int i;
for (i=0; i<line->n_info; i++)
if ( line->d.info[i].key == fmt->id ) break;
fmt->ready = 1;
}
+static void process_complete_format(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
+{
+ if ( convert->nsamples )
+ {
+ int i,j;
+ if ( line->n_fmt)
+ {
+ int gt_i = -1;
+ bcf_fmt_t *fmt = line->d.fmt;
+ int first = 1;
+ for (i=0; i<(int)line->n_fmt; i++)
+ {
+ if ( !fmt[i].p || fmt[i].id<0 ) continue;
+ if ( !first ) kputc(':', str);
+ first = 0;
+ kputs(convert->header->id[BCF_DT_ID][fmt[i].id].key, str);
+ if ( strcmp(convert->header->id[BCF_DT_ID][fmt[i].id].key, "GT") == 0) gt_i = i;
+ }
+ if ( first ) kputc('.', str);
+ for (j=0; j<convert->nsamples; j++)
+ {
+ kputc('\t', str);
+ first = 1;
+ for (i=0; i<(int)line->n_fmt; i++)
+ {
+ bcf_fmt_t *f = &fmt[i];
+ if ( !f->p ) continue;
+ if ( !first ) kputc(':', str);
+ first = 0;
+ if (gt_i == i)
+ bcf_format_gt(f,convert->samples[j],str);
+ else
+ bcf_fmt_array(str, f->n, f->type, f->p + convert->samples[j] * f->size);
+ }
+ if ( first ) kputc('.', str);
+ }
+ }
+ else
+ for (j=0; j<=line->n_sample; j++)
+ kputs("\t.", str);
+ }
+ else
+ kputc('.',str);
+}
static void process_format(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
{
if ( !fmt->ready )
if ( line_type & VCF_INDEL ) { if (i) kputc(',',str); kputs("INDEL", str); i++; }
if ( line_type & VCF_OTHER ) { if (i) kputc(',',str); kputs("OTHER", str); i++; }
if ( line_type & VCF_BND ) { if (i) kputc(',',str); kputs("BND", str); i++; }
+ if ( line_type & VCF_OVERLAP ) { if (i) kputc(',',str); kputs("OVERLAP", str); i++; }
}
static void process_line(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
{
// for (i=0; i<convert->nsamples; i++) kputs(" 0.33 0.33 0.33", str);
// return;
- error("Error parsing GT tag at %s:%d\n", bcf_seqname(convert->header,line),line->pos+1);
+ error("Error parsing GT tag at %s:%"PRId64"\n", bcf_seqname(convert->header,line),(int64_t) line->pos+1);
}
n /= convert->nsamples;
// for (i=0; i<convert->nsamples; i++) kputs(" 0.33 0.33 0.33", str);
// return;
- error("Error parsing PL tag at %s:%d\n", bcf_seqname(convert->header,line),line->pos+1);
+ error("Error parsing PL tag at %s:%"PRId64"\n", bcf_seqname(convert->header,line),(int64_t) line->pos+1);
}
n /= convert->nsamples;
// for (i=0; i<convert->nsamples; i++) kputs(" 0.33 0.33 0.33", str);
// return;
- error("Error parsing GP tag at %s:%d\n", bcf_seqname(convert->header,line),line->pos+1);
+ error("Error parsing GP tag at %s:%"PRId64"\n", bcf_seqname(convert->header,line),(int64_t) line->pos+1);
}
n /= convert->nsamples;
{
if ( ptr[j]==bcf_int32_vector_end ) break;
if ( ptr[j]==bcf_int32_missing ) { ptr[j]=0; continue; }
- if ( ptr[j]<0 || ptr[j]>1 ) error("[%s:%d:%f] GP value outside range [0,1]; bcftools convert expects the VCF4.3+ spec for the GP field encoding genotype posterior probabilities", bcf_seqname(convert->header,line),line->pos+1,ptr[j]);
+ if ( ptr[j]<0 || ptr[j]>1 ) error("[%s:%"PRId64":%f] GP value outside range [0,1]; bcftools convert expects the VCF4.3+ spec for the GP field encoding genotype posterior probabilities", bcf_seqname(convert->header,line),(int64_t) line->pos+1,ptr[j]);
sum+=ptr[j];
}
if ( j==line->n_allele )
int i, gt_id = bcf_hdr_id2int(convert->header, BCF_DT_ID, "GT");
if ( !bcf_hdr_idinfo_exists(convert->header,BCF_HL_FMT,gt_id) )
- error("FORMAT/GT tag not present at %s:%d\n", bcf_seqname(convert->header, line), line->pos+1);
+ error("FORMAT/GT tag not present at %s:%"PRId64"\n", bcf_seqname(convert->header, line),(int64_t) line->pos+1);
if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);
bcf_fmt_t *fmt_gt = NULL;
for (i=0; i<line->n_fmt; i++)
if ( line->d.fmt[i].id==gt_id ) { fmt_gt = &line->d.fmt[i]; break; }
if ( !fmt_gt )
- error("FORMAT/GT tag not present at %s:%d\n", bcf_seqname(convert->header, line), line->pos+1);
+ error("FORMAT/GT tag not present at %s:%"PRId64"\n", bcf_seqname(convert->header, line),(int64_t) line->pos+1);
// Alloc all memory in advance to avoid kput routines. The biggest allowed allele index is 99
if ( line->n_allele > 100 )
- error("Too many alleles (%d) at %s:%d\n", line->n_allele, bcf_seqname(convert->header, line), line->pos+1);
+ error("Too many alleles (%d) at %s:%"PRId64"\n", line->n_allele, bcf_seqname(convert->header, line),(int64_t) line->pos+1);
if ( ks_resize(str, str->l+convert->nsamples*8) != 0 )
- error("Could not alloc %"PRIu64" bytes\n", (uint64_t)(str->l + convert->nsamples*8));
+ error("Could not alloc %" PRIu64 " bytes\n", (uint64_t)(str->l + convert->nsamples*8));
if ( fmt_gt->type!=BCF_BT_INT8 ) // todo: use BRANCH_INT if the VCF is valid
- error("Uh, too many alleles (%d) or redundant BCF representation at %s:%d\n", line->n_allele, bcf_seqname(convert->header, line), line->pos+1);
+ error("Uh, too many alleles (%d) or redundant BCF representation at %s:%"PRId64"\n", line->n_allele, bcf_seqname(convert->header, line),(int64_t) line->pos+1);
if ( fmt_gt->n!=1 && fmt_gt->n!=2 )
- error("Uh, ploidy of %d not supported, see %s:%d\n", fmt_gt->n, bcf_seqname(convert->header, line), line->pos+1);
+ error("Uh, ploidy of %d not supported, see %s:%"PRId64"\n", fmt_gt->n, bcf_seqname(convert->header, line),(int64_t) line->pos+1);
int8_t *ptr = ((int8_t*) fmt_gt->p) - fmt_gt->n;
for (i=0; i<convert->nsamples; i++)
int i, gt_id = bcf_hdr_id2int(convert->header, BCF_DT_ID, "GT");
if ( !bcf_hdr_idinfo_exists(convert->header,BCF_HL_FMT,gt_id) )
- error("FORMAT/GT tag not present at %s:%d\n", bcf_seqname(convert->header, line), line->pos+1);
+ error("FORMAT/GT tag not present at %s:%"PRId64"\n", bcf_seqname(convert->header, line),(int64_t) line->pos+1);
if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);
bcf_fmt_t *fmt_gt = NULL;
for (i=0; i<line->n_fmt; i++)
if ( line->d.fmt[i].id==gt_id ) { fmt_gt = &line->d.fmt[i]; break; }
if ( !fmt_gt )
- error("FORMAT/GT tag not present at %s:%d\n", bcf_seqname(convert->header, line), line->pos+1);
+ error("FORMAT/GT tag not present at %s:%"PRId64"\n", bcf_seqname(convert->header, line),(int64_t) line->pos+1);
// Alloc all memory in advance to avoid kput routines. The biggest allowed allele index is 99
if ( line->n_allele > 100 )
- error("Too many alleles (%d) at %s:%d\n", line->n_allele, bcf_seqname(convert->header, line), line->pos+1);
+ error("Too many alleles (%d) at %s:%"PRId64"\n", line->n_allele, bcf_seqname(convert->header, line),(int64_t) line->pos+1);
if ( ks_resize(str, str->l+convert->nsamples*8) != 0 )
- error("Could not alloc %"PRIu64" bytes\n", (uint64_t)(str->l + convert->nsamples*8));
+ error("Could not alloc %" PRIu64 " bytes\n", (uint64_t)(str->l + convert->nsamples*8));
if ( fmt_gt->type!=BCF_BT_INT8 ) // todo: use BRANCH_INT if the VCF is valid
- error("Uh, too many alleles (%d) or redundant BCF representation at %s:%d\n", line->n_allele, bcf_seqname(convert->header, line), line->pos+1);
+ error("Uh, too many alleles (%d) or redundant BCF representation at %s:%"PRId64"\n", line->n_allele, bcf_seqname(convert->header, line),(int64_t) line->pos+1);
int8_t *ptr = ((int8_t*) fmt_gt->p) - fmt_gt->n;
for (i=0; i<convert->nsamples; i++)
str->s[--str->l] = 0; // delete the last space
}
+static void process_rsid_hex(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
+{
+ char *ptr = line->d.id;
+ ptr += 2; // remove 'rs'
+ ksprintf(str, "%08" PRIx32 "", (uint32_t)strtoul(ptr, NULL, 10));
+}
+
+static void process_variantkey_hex(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
+{
+ uint64_t vk = variantkey(
+ convert->header->id[BCF_DT_CTG][line->rid].key,
+ strlen(convert->header->id[BCF_DT_CTG][line->rid].key),
+ line->pos,
+ line->d.allele[0],
+ strlen(line->d.allele[0]),
+ line->d.allele[1],
+ strlen(line->d.allele[1]));
+ ksprintf(str, "%016" PRIx64 "", vk);
+}
+
+static void process_pbinom(convert_t *convert, bcf1_t *line, fmt_t *fmt, int isample, kstring_t *str)
+{
+ int i;
+ if ( !fmt->ready )
+ {
+ fmt->fmt = NULL; // AD
+ fmt->usr = NULL; // GT
+
+ for (i=0; i<(int)line->n_fmt; i++)
+ if ( line->d.fmt[i].id==fmt->id ) { fmt->fmt = &line->d.fmt[i]; break; }
+
+ // Check that the first field is GT
+ int gt_id = bcf_hdr_id2int(convert->header, BCF_DT_ID, "GT");
+ if ( !bcf_hdr_idinfo_exists(convert->header, BCF_HL_FMT, fmt->id) ) error("Error: FORMAT/GT is not defined in the header\n");
+ for (i=0; i<(int)line->n_fmt; i++)
+ if ( line->d.fmt[i].id==gt_id ) { fmt->usr = &line->d.fmt[i]; break; } // it should always be first according to VCF spec, but...
+
+ if ( fmt->usr && line->d.fmt[i].type!=BCF_BT_INT8 ) // skip sites with many alleles
+ fmt->usr = NULL;
+
+ fmt->ready = 1;
+ }
+ bcf_fmt_t *gt_fmt = (bcf_fmt_t*) fmt->usr;
+ if ( !fmt->fmt || !gt_fmt || gt_fmt->n!=2 ) goto invalid;
+
+ int n[2] = {0,0};
+ int8_t *gt = (int8_t*)(gt_fmt->p + isample*gt_fmt->size);
+ for (i=0; i<2; i++)
+ {
+ if ( bcf_gt_is_missing(gt[i]) || gt[i] == bcf_int8_vector_end ) goto invalid;
+ int al = bcf_gt_allele(gt[i]);
+ if ( al > line->n_allele || al >= fmt->fmt->n ) goto invalid;
+
+ #define BRANCH(type_t, missing, vector_end) { \
+ type_t val = ((type_t *) fmt->fmt->p)[al + isample*fmt->fmt->n]; \
+ if ( val==missing || val==vector_end ) goto invalid; \
+ else n[i] = val; \
+ }
+ switch (fmt->fmt->type)
+ {
+ case BCF_BT_INT8: BRANCH(int8_t, bcf_int8_missing, bcf_int8_vector_end); break;
+ case BCF_BT_INT16: BRANCH(int16_t, bcf_int16_missing, bcf_int16_vector_end); break;
+ case BCF_BT_INT32: BRANCH(int32_t, bcf_int32_missing, bcf_int32_vector_end); break;
+ default: goto invalid; break;
+ }
+ #undef BRANCH
+ }
+
+ if ( n[0]==n[1] ) kputc(n[0]==0 ? '.':'0', str);
+ else
+ {
+ double pval = n[0] < n[1] ? kf_betai(n[1], n[0] + 1, 0.5) : kf_betai(n[0], n[1] + 1, 0.5);
+ pval *= 2;
+ assert( pval-1 < 1e-10 );
+ if ( pval>=1 ) pval = 0; // this can happen, machine precision error, eg. kf_betai(1,0,0.5)
+ else
+ pval = -4.34294481903*log(pval);
+ kputd(pval, str);
+ }
+ return;
+
+invalid:
+ kputc('.', str);
+}
+
static fmt_t *register_tag(convert_t *convert, int type, char *key, int is_gtf)
{
convert->nfmt++;
else if ( !strcmp("QUAL",key) ) { fmt->type = T_QUAL; }
else if ( !strcmp("FILTER",key) ) { fmt->type = T_FILTER; }
else if ( !strcmp("_CHROM_POS_ID",key) ) { fmt->type = T_CHROM_POS_ID; }
- else if ( id>=0 && bcf_hdr_idinfo_exists(convert->header,BCF_HL_INFO,id) )
- {
- fmt->type = T_INFO;
- fprintf(bcftools_stderr,"Warning: Assuming INFO/%s\n", key);
- }
+ else if ( !strcmp("RSX",key) ) { fmt->type = T_RSX; }
+ else if ( !strcmp("VKX",key) ) { fmt->type = T_VKX; }
+ else if ( id>=0 && bcf_hdr_idinfo_exists(convert->header,BCF_HL_INFO,id) ) { fmt->type = T_INFO; }
+ }
+ if ( fmt->type==T_PBINOM )
+ {
+ fmt->id = bcf_hdr_id2int(convert->header, BCF_DT_ID, fmt->key);
+ if ( !bcf_hdr_idinfo_exists(convert->header,BCF_HL_FMT, fmt->id) ) error("No such FORMAT tag defined in the header: %s\n", fmt->key);
}
}
case T_CHROM: fmt->handler = &process_chrom; break;
case T_POS: fmt->handler = &process_pos; break;
case T_POS0: fmt->handler = &process_pos0; break;
- case T_END: fmt->handler = &process_end; break;
- case T_END0: fmt->handler = &process_end0; break;
+ case T_END: fmt->handler = &process_end; convert->max_unpack |= BCF_UN_INFO; break;
+ case T_END0: fmt->handler = &process_end0; convert->max_unpack |= BCF_UN_INFO; break;
case T_ID: fmt->handler = &process_id; break;
case T_REF: fmt->handler = &process_ref; break;
case T_ALT: fmt->handler = &process_alt; break;
case T_QUAL: fmt->handler = &process_qual; break;
case T_FILTER: fmt->handler = &process_filter; convert->max_unpack |= BCF_UN_FLT; break;
case T_INFO: fmt->handler = &process_info; convert->max_unpack |= BCF_UN_INFO; break;
- case T_FORMAT: fmt->handler = &process_format; convert->max_unpack |= BCF_UN_FMT; break;
+ case T_FORMAT: fmt->handler = fmt->key ? &process_format : &process_complete_format; convert->max_unpack |= BCF_UN_FMT; break;
case T_SAMPLE: fmt->handler = &process_sample; break;
case T_SEP: fmt->handler = &process_sep; break;
case T_IS_TS: fmt->handler = &process_is_ts; break;
case T_GT_TO_HAP2: fmt->handler = &process_gt_to_hap2; convert->max_unpack |= BCF_UN_FMT; break;
case T_TBCSQ: fmt->handler = &process_tbcsq; fmt->destroy = &destroy_tbcsq; convert->max_unpack |= BCF_UN_FMT; break;
case T_LINE: fmt->handler = &process_line; convert->max_unpack |= BCF_UN_FMT; break;
+ case T_RSX: fmt->handler = &process_rsid_hex; break;
+ case T_VKX: fmt->handler = &process_variantkey_hex; break;
+ case T_PBINOM: fmt->handler = &process_pbinom; convert->max_unpack |= BCF_UN_FMT; break;
default: error("TODO: handler for type %d\n", fmt->type);
}
if ( key && fmt->type==T_INFO )
else if ( !strcmp(str.s, "IUPACGT") ) register_tag(convert, T_IUPAC_GT, "GT", is_gtf);
else if ( !strcmp(str.s, "INFO") )
{
- if ( *q!='/' ) error("Could not parse format string: %s\n", convert->format_str);
+ if ( *q!='/' )
+ {
+ int id = bcf_hdr_id2int(convert->header, BCF_DT_ID, str.s);
+ if ( bcf_hdr_idinfo_exists(convert->header,BCF_HL_INFO,id) )
+ error("Could not parse format string \"%s\". Did you mean %%INFO/%s?\n", convert->format_str,str.s);
+ else
+ error("Could not parse format string: %s\n", convert->format_str);
+ }
p = ++q;
str.l = 0;
while ( *q && (isalnum(*q) || *q=='_' || *q=='.') ) q++;
fmt_t *fmt = register_tag(convert, T_INFO, str.s, is_gtf);
fmt->subscript = parse_subscript(&q);
}
+ else if ( !strcmp(str.s,"PBINOM") )
+ {
+ if ( *q!='(' ) error("Could not parse the expression: %s\n", convert->format_str);
+ p = ++q;
+ str.l = 0;
+ while ( *q && *q!=')' ) q++;
+ if ( q-p==0 ) error("Could not parse format string: %s\n", convert->format_str);
+ kputsn(p, q-p, &str);
+ register_tag(convert, T_PBINOM, str.s, is_gtf);
+ q++;
+ }
else
{
fmt_t *fmt = register_tag(convert, T_FORMAT, str.s, is_gtf);
else if ( !strcmp(str.s, "_GP_TO_PROB3") ) register_tag(convert, T_GP_TO_PROB3, str.s, is_gtf);
else if ( !strcmp(str.s, "_GT_TO_HAP") ) register_tag(convert, T_GT_TO_HAP, str.s, is_gtf);
else if ( !strcmp(str.s, "_GT_TO_HAP2") ) register_tag(convert, T_GT_TO_HAP2, str.s, is_gtf);
+ else if ( !strcmp(str.s, "RSX") ) register_tag(convert, T_RSX, str.s, is_gtf);
+ else if ( !strcmp(str.s, "VKX") ) register_tag(convert, T_VKX, str.s, is_gtf);
+ else if ( !strcmp(str.s,"pbinom") ) error("Error: pbinom() is currently supported only with FORMAT tags. (todo)\n");
else if ( !strcmp(str.s, "INFO") )
{
- if ( *q!='/' ) error("Could not parse format string: %s\n", convert->format_str);
- p = ++q;
- str.l = 0;
- while ( *q && (isalnum(*q) || *q=='_' || *q=='.') ) q++;
- if ( q-p==0 ) error("Could not parse format string: %s\n", convert->format_str);
- kputsn(p, q-p, &str);
- fmt_t *fmt = register_tag(convert, T_INFO, str.s, is_gtf);
- fmt->subscript = parse_subscript(&q);
+ if ( *q=='/' )
+ {
+ p = ++q;
+ str.l = 0;
+ while ( *q && (isalnum(*q) || *q=='_' || *q=='.') ) q++;
+ if ( q-p==0 ) error("Could not parse format string: %s\n", convert->format_str);
+ kputsn(p, q-p, &str);
+ fmt_t *fmt = register_tag(convert, T_INFO, str.s, is_gtf);
+ fmt->subscript = parse_subscript(&q);
+ }
+ else
+ register_tag(convert, T_INFO, NULL, is_gtf); // the whole INFO
}
+ else if ( !strcmp(str.s, "FORMAT") )
+ register_tag(convert, T_FORMAT, NULL, 0);
else
{
fmt_t *fmt = register_tag(convert, T_INFO, str.s, is_gtf);
int convert_line(convert_t *convert, bcf1_t *line, kstring_t *str)
{
if ( !convert->allow_undef_tags && convert->undef_info_tag )
- error("Error: no such tag defined in the VCF header: INFO/%s. FORMAT fields must be in square brackets, e.g. \"[ %s]\"\n", convert->undef_info_tag,convert->undef_info_tag);
+ {
+ kstring_t msg = {0,0,0};
+ ksprintf(&msg,"Error: no such tag defined in the VCF header: INFO/%s", convert->undef_info_tag);
+
+ int hdr_id = bcf_hdr_id2int(convert->header,BCF_DT_ID,convert->undef_info_tag);
+ if ( hdr_id>=0 && bcf_hdr_idinfo_exists(convert->header,BCF_HL_FMT,hdr_id) )
+ ksprintf(&msg,". FORMAT fields must be enclosed in square brackets, e.g. \"[ %%%s]\"", convert->undef_info_tag);
+ error("%s\n", msg.s);
+ }
int l_ori = str->l;
bcf_unpack(line, convert->max_unpack);
for (js=0; js<convert->nsamples; js++)
{
// Skip samples when filtering was requested
- if ( *convert->subset_samples && !(*convert->subset_samples)[js] ) continue;
+ if ( convert->subset_samples && *convert->subset_samples && !(*convert->subset_samples)[js] ) continue;
// Here comes a hack designed for TBCSQ. When running on large files,
// such as 1000GP, there are too many empty fields in the output and
+//$bt csq -f $ref -g $gff -p r -Ou -o /dev/null /lustre/scratch116/vr/projects/g1k/phase3/release/ALL.chr4.phase3_shapeit2_mvncall_integrated_v5a.20130502.genotypes.vcf.gz
+
+
/* The MIT License
Copyright (c) 2016-2018 Genome Research Ltd.
*/
/*
Things that would be nice to have
+ - dynamic N_REF_PAD
- for stop-lost events (also in frameshifts) report the number of truncated aa's
- memory could be greatly reduced by indexing gff (but it is quite compact already)
- deletions that go beyond transcript boundaries are not checked at sequence level
splice_region_variant .. change within 1-3 bases of the exon or 3-8 bases of the intron
synonymous_variant .. DNA sequence variant resulting in no amino acid change
stop_retained_variant .. different stop codon
+ start_retained_variant .. start codon retained by indel realignment
non_coding_variant .. variant in non-coding sequence, such as RNA gene
5_prime_UTR_variant
3_prime_UTR_variant
#include <stdlib.h>
#include <getopt.h>
#include <math.h>
+#include <inttypes.h>
#include <htslib/hts.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/faidx.h>
#include <errno.h>
#include <unistd.h>
-#include <stdint.h>
#include <ctype.h>
#include "bcftools.h"
#include "filter.h"
#define CSQ_UPSTREAM_STOP (1<<19) // adds * in front of the csq string
#define CSQ_INCOMPLETE_CDS (1<<20) // to remove START/STOP in incomplete CDS, see ENSG00000173376/synon.vcf
#define CSQ_CODING_SEQUENCE (1<<21) // cannot tell exactly what it is, but it does affect the coding sequence
+#define CSQ_ELONGATION (1<<22) // symbolic insertion
+#define CSQ_START_RETAINED (1<<23)
// Haplotype-aware consequences, printed in one vcf record only, the rest has a reference @12345
#define CSQ_COMPOUND (CSQ_SYNONYMOUS_VARIANT|CSQ_MISSENSE_VARIANT|CSQ_STOP_LOST|CSQ_STOP_GAINED| \
CSQ_INFRAME_DELETION|CSQ_INFRAME_INSERTION|CSQ_FRAMESHIFT_VARIANT| \
CSQ_START_LOST|CSQ_STOP_RETAINED|CSQ_INFRAME_ALTERING|CSQ_INCOMPLETE_CDS| \
- CSQ_UPSTREAM_STOP)
-#define CSQ_START_STOP (CSQ_STOP_LOST|CSQ_STOP_GAINED|CSQ_STOP_RETAINED|CSQ_START_LOST)
+ CSQ_UPSTREAM_STOP|CSQ_START_RETAINED)
+#define CSQ_START_STOP (CSQ_STOP_LOST|CSQ_STOP_GAINED|CSQ_STOP_RETAINED|CSQ_START_LOST|CSQ_START_RETAINED)
#define CSQ_PRN_STRAND(csq) ((csq)&CSQ_COMPOUND && !((csq)&(CSQ_SPLICE_ACCEPTOR|CSQ_SPLICE_DONOR|CSQ_SPLICE_REGION)))
#define CSQ_PRN_TSCRIPT (~(CSQ_INTRON|CSQ_NON_CODING))
"inframe_altering",
NULL,
NULL,
- "coding_sequence"
+ "coding_sequence",
+ "feature_elongation",
+ "start_retained"
};
typedef struct
{
char *name; // human readable name, e.g. ORF45
- uint8_t iseq;
+ uint32_t iseq;
}
gf_gene_t;
typedef struct
{
bcf1_t *line;
uint32_t *smpl; // bitmask of sample consequences with first/second haplotype interleaved
- uint32_t nfmt:4, nvcsq:28, mvcsq;
+ uint32_t nfmt:4, // the bitmask size (the number of integers per sample)
+ nvcsq:28, mvcsq;
vcsq_t *vcsq; // there can be multiple consequences for a single VCF record
}
vrec_t;
{
vrec_t **vrec; // buffer of VCF lines with the same position
int n, m;
+ uint32_t keep_until; // the maximum transcript end position
};
KHASH_MAP_INIT_INT(pos2vbuf, vbuf_t*)
char *outdir, **argv, *fa_fname, *gff_fname, *output_fname;
char *bcsq_tag;
int argc, output_type;
- int phase, quiet, local_csq;
+ int phase, verbosity, local_csq, record_cmd_line;
int ncsq_max, nfmt_bcsq; // maximum number of csq per site that can be accessed from FORMAT/BCSQ
int ncsq_small_warned;
+ int brief_predictions;
int rid; // current chromosome
tr_heap_t *active_tr; // heap of active transcripts for quick flushing
int ncsq_buf, mcsq_buf;
id_tbl_t tscript_ids; // mapping between transcript id (eg. Zm00001d027245_T001) and a numeric idx
int force; // force run under various conditions. Currently only to skip out-of-phase transcripts
+ int n_threads; // extra compression/decompression threads
faidx_t *fai;
kstring_t str, str2;
aux->seq[aux->nseq] = strdup(chr_beg);
iseq = khash_str2int_inc(aux->seq2int, aux->seq[aux->nseq]);
aux->nseq++;
- assert( aux->nseq < 256 ); // see gf_gene_t.iseq
+ assert( aux->nseq < 1<<29 ); // see gf_gene_t.iseq and ftr_t.iseq
}
chr_end[1] = c;
return iseq;
int biotype = gff_parse_biotype(ss);
if ( biotype <= 0 )
{
- if ( !gff_ignored_biotype(args, ss) && args->quiet<2 ) fprintf(stderr,"ignored transcript: %s\n",line);
+ if ( !gff_ignored_biotype(args, ss) && args->verbosity > 0 ) fprintf(stderr,"ignored transcript: %s\n",line);
return;
}
int biotype = gff_parse_biotype(ss);
if ( biotype <= 0 )
{
- if ( !gff_ignored_biotype(args, ss) && args->quiet<2 ) fprintf(stderr,"ignored gene: %s\n",line);
+ if ( !gff_ignored_biotype(args, ss) && args->verbosity > 0 ) fprintf(stderr,"ignored gene: %s\n",line);
return;
}
if ( !ss ) return -1; // no ID, ignore the line
if ( !strncmp("chromosome",ss+3,10) ) return -1;
if ( !strncmp("supercontig",ss+3,11) ) return -1;
- if ( args->quiet<2 ) fprintf(stderr,"ignored: %s\n", line);
+ if ( args->verbosity > 0 ) fprintf(stderr,"ignored: %s\n", line);
return -1;
}
// 7. column: strand
if ( *ss == '+' ) ftr->strand = STRAND_FWD;
else if ( *ss == '-' ) ftr->strand = STRAND_REV;
- else { if ( args->quiet<2 ) fprintf(stderr,"Skipping unknown strand: %c\n", *ss); return -1; }
+ else { if ( args->verbosity > 0 ) fprintf(stderr,"Skipping unknown strand: %c\n", *ss); return -1; }
ss += 2;
// 8. column: phase (codon offset)
else if ( *ss == '1' ) ftr->phase = 1;
else if ( *ss == '2' ) ftr->phase = 2;
else if ( *ss == '.' ) ftr->phase = 0; // exons do not have phase
- else { if ( args->quiet<2 ) fprintf(stderr,"Skipping unknown phase: %c, %s\n", *ss, line); return -1; }
+ else { if ( args->verbosity > 0 ) fprintf(stderr,"Skipping unknown phase: %c, %s\n", *ss, line); return -1; }
ss += 2;
// substring search for "Parent=transcript:ENST00000437963"
{
if ( args->force )
{
- if ( args->quiet < 2 )
+ if ( args->verbosity > 0 )
fprintf(stderr,"Warning: GFF3 assumption failed for transcript %s, CDS=%d: phase!=len%%3 (phase=%d, len=%d)\n",args->tscript_ids.str[tr->id],tr->cds[i]->beg+1,phase,len);
tscript_ok = 0;
break;
{
if ( args->force )
{
- if ( args->quiet < 2 )
+ if ( args->verbosity > 0 )
fprintf(stderr,"Warning: GFF3 assumption failed for transcript %s, CDS=%d: phase!=len%%3 (phase=%d, len=%d)\n",args->tscript_ids.str[tr->id],tr->cds[i]->beg+1,phase,len);
tscript_ok = 0;
break;
}
tscript_init_cds(args);
- if ( !args->quiet )
+ if ( args->verbosity > 0 )
{
fprintf(stderr,"Indexed %d transcripts, %d exons, %d CDSs, %d UTRs\n",
regidx_nregs(args->idx_tscript),
free(aux->seq);
gff_id_destroy(&aux->gene_ids);
- if ( args->quiet<2 && khash_str2int_size(aux->ignored_biotypes) )
+ if ( args->verbosity > 0 && khash_str2int_size(aux->ignored_biotypes) )
{
khash_t(str2int) *ign = (khash_t(str2int)*)aux->ignored_biotypes;
fprintf(stderr,"Ignored the following biotypes:\n");
for (i = kh_begin(ign); i < kh_end(ign); i++)
{
if ( !kh_exist(ign,i)) continue;
- fprintf(stderr,"\t%dx\t.. %s\n", kh_value(ign,i), kh_key(ign,i));
+ const char *biotype = kh_key(ign,i);
+ if ( !strcmp(biotype,"TCE") ) biotype = "TCE (\"To be Experimentally Confirmed\")";
+ fprintf(stderr,"\t%dx\t.. %s\n", kh_value(ign,i), biotype);
}
}
khash_str2int_destroy_free(aux->ignored_biotypes);
{
args->nfmt_bcsq = 1 + (args->ncsq_max - 1) / 32;
- if ( !args->quiet ) fprintf(stderr,"Parsing %s ...\n", args->gff_fname);
+ if ( args->verbosity > 0 ) fprintf(stderr,"Parsing %s ...\n", args->gff_fname);
init_gff(args);
args->rid = -1;
if ( args->output_type==FT_TAB_TEXT )
{
// significant speedup for plain VCFs
- bcf_hdr_set_samples(args->hdr,NULL,0);
+ if (bcf_hdr_set_samples(args->hdr,NULL,0) < 0)
+ error_errno("[%s] Couldn't build sample filter", __func__);
}
args->phase = PHASE_DROP_GT;
}
if ( args->output_type==FT_TAB_TEXT )
{
args->out = args->output_fname ? fopen(args->output_fname,"w") : stdout;
- if ( !args->out ) error("Failed to open %s: %s\n", args->output_fname,strerror(errno));
+ if ( !args->out ) error("Failed to write to %s: %s\n", !strcmp("-",args->output_fname)?"standard output":args->output_fname,strerror(errno));
fprintf(args->out,"# This file was produced by: bcftools +csq(%s+htslib-%s)\n", bcftools_version(),hts_version());
fprintf(args->out,"# The command line was:\tbcftools +%s", args->argv[0]);
else
{
args->out_fh = hts_open(args->output_fname? args->output_fname : "-",hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to %s: %s\n", args->output_fname? args->output_fname : "standard output", strerror(errno));
- bcf_hdr_append_version(args->hdr,args->argc,args->argv,"bcftools/csq");
- bcf_hdr_printf(args->hdr,"##INFO=<ID=%s,Number=.,Type=String,Description=\"%s consequence annotation from BCFtools/csq. Format: '[*]consequence|gene|transcript|biotype[|strand|amino_acid_change|dna_change]' or, for consequences of variants split across multiple sites, a pointer to the record storing the consequences '@position'. '*' prefix indicates a consequence downstream from a stop \">",args->bcsq_tag, args->local_csq ? "Local" : "Haplotype-aware");
+ if ( args->out_fh == NULL ) error("[%s] Error: cannot write to %s: %s\n", __func__,args->output_fname? args->output_fname : "standard output", strerror(errno));
+ if ( args->n_threads > 0)
+ hts_set_opt(args->out_fh, HTS_OPT_THREAD_POOL, args->sr->p);
+ if ( args->record_cmd_line ) bcf_hdr_append_version(args->hdr,args->argc,args->argv,"bcftools/csq");
+ bcf_hdr_printf(args->hdr,"##INFO=<ID=%s,Number=.,Type=String,Description=\"%s consequence annotation from BCFtools/csq, see http://samtools.github.io/bcftools/howtos/csq-calling.html for details. Format: Consequence|gene|transcript|biotype|strand|amino_acid_change|dna_change\">",args->bcsq_tag, args->local_csq ? "Local" : "Haplotype-aware");
if ( args->hdr_nsmpl )
bcf_hdr_printf(args->hdr,"##FORMAT=<ID=%s,Number=.,Type=Integer,Description=\"Bitmask of indexes to INFO/BCSQ, with interleaved first/second haplotype. Use \\\"bcftools query -f'[%%CHROM\\t%%POS\\t%%SAMPLE\\t%%TBCSQ\\n]'\\\" to translate.\">",args->bcsq_tag);
- bcf_hdr_write(args->out_fh, args->hdr);
+ if ( bcf_hdr_write(args->out_fh, args->hdr)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname?args->output_fname:"standard output");
}
- if ( !args->quiet ) fprintf(stderr,"Calling...\n");
+ if ( args->verbosity > 0 ) fprintf(stderr,"Calling...\n");
}
void destroy_data(args_t *args)
splice->vcf.pos = rec->pos;
splice->vcf.rlen = rec->rlen;
splice->vcf.ref = rec->d.allele[0];
+ splice->csq = 0;
}
static inline void splice_build_hap(splice_t *splice, uint32_t beg, int len)
{
#endif
}
void csq_stage(args_t *args, csq_t *csq, bcf1_t *rec);
-static inline int csq_stage_utr(args_t *args, regitr_t *itr, bcf1_t *rec, uint32_t trid)
+static inline int csq_stage_utr(args_t *args, regitr_t *itr, bcf1_t *rec, uint32_t trid, uint32_t type)
{
while ( regitr_overlap(itr) )
{
csq_t csq;
memset(&csq, 0, sizeof(csq_t));
csq.pos = rec->pos;
- csq.type.type = utr->which==prime5 ? CSQ_UTR5 : CSQ_UTR3;
+ csq.type.type = (utr->which==prime5 ? CSQ_UTR5 : CSQ_UTR3) | type;
csq.type.biotype = tr->type;
csq.type.strand = tr->strand;
csq.type.trid = tr->id;
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,splice->ref_beg+1,splice->ref_beg+1, itr) ) // adjacent utr
{
- ret = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ ret = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
if ( ret!=0 )
{
regitr_destroy(itr);
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,splice->ref_end-1,splice->ref_end-1, itr) ) // adjacent utr
{
- ret = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ ret = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
if ( ret!=0 )
{
regitr_destroy(itr);
return SPLICE_INSIDE;
}
+int shifted_del_synonymous(args_t *args, splice_t *splice, uint32_t ex_beg, uint32_t ex_end)
+{
+ static int small_ref_padding_warned = 0;
+ tscript_t *tr = splice->tr;
+
+ // We know the VCF record overlaps the exon, but does it overlap the start codon?
+ if ( tr->strand==STRAND_REV && splice->vcf.pos + splice->vcf.rlen + 2 <= ex_end ) return 0;
+ if ( tr->strand==STRAND_FWD && splice->vcf.pos >= ex_beg + 3 ) return 0;
+
+#if XDBG
+ fprintf(stderr,"shifted_del_synonymous: %d-%d %s\n",ex_beg,ex_end, tr->strand==STRAND_FWD?"fwd":"rev");
+ fprintf(stderr," %d .. %s > %s\n",splice->vcf.pos+1,splice->vcf.ref,splice->vcf.alt);
+#endif
+
+ // is there enough ref sequence for the extension? All coordinates are 0-based
+ int ref_len = strlen(splice->vcf.ref);
+ int alt_len = strlen(splice->vcf.alt);
+ assert( ref_len > alt_len );
+ int ndel = ref_len - alt_len;
+
+ if ( tr->strand==STRAND_REV )
+ {
+ int32_t vcf_ref_end = splice->vcf.pos + ref_len - 1; // end pos of the VCF REF allele
+ int32_t tr_ref_end = splice->tr->end + N_REF_PAD; // the end pos of accessible cached ref seq
+ if ( vcf_ref_end + ndel > tr_ref_end )
+ {
+ if ( !small_ref_padding_warned )
+ {
+ fprintf(stderr,"Warning: Could not verify synonymous start/stop at %s:%d due to small N_REF_PAD. (Improve me?)\n",bcf_seqname(args->hdr,splice->vcf.rec),splice->vcf.pos+1);
+ small_ref_padding_warned = 1;
+ }
+ return 0;
+ }
+
+ char *ptr_vcf = splice->vcf.ref + alt_len; // the first deleted base in the VCF REF allele
+ char *ptr_ref = splice->tr->ref + N_REF_PAD + (vcf_ref_end + 1 - splice->tr->beg); // the first ref base after the ndel bases deleted
+#if XDBG
+ fprintf(stderr,"vcf: %s\nref: %s\n",ptr_vcf,ptr_ref);
+#endif
+ int i = 0;
+ while ( ptr_vcf[i] && ptr_vcf[i]==ptr_ref[i] ) i++;
+ if ( ptr_vcf[i] ) return 0; // the deleted sequence cannot be replaced
+ }
+ else
+ {
+ // STRAND_FWD
+ int32_t vcf_block_beg = splice->vcf.pos + ref_len - 2*ndel; // the position of the first base of the ref block that could potentially replace the deletion
+ if ( vcf_block_beg < 0 ) return 0;
+
+#if XDBG
+ fprintf(stderr,"vcf_block_beg: %d\n",vcf_block_beg+1);
+#endif
+
+ if ( N_REF_PAD + vcf_block_beg < ex_beg )
+ {
+ if ( !small_ref_padding_warned )
+ {
+ fprintf(stderr,"Warning: Could not verify synonymous start/stop at %s:%d due to small N_REF_PAD. (Improve me?)\n",bcf_seqname(args->hdr,splice->vcf.rec),splice->vcf.pos+1);
+ small_ref_padding_warned = 1;
+ }
+ return 0;
+ }
+
+ char *ptr_vcf = splice->vcf.ref + alt_len; // the first deleted base in the VCF REF allele
+ char *ptr_ref = splice->tr->ref + N_REF_PAD + vcf_block_beg - splice->tr->beg; // the replacement ref block
+#if XDBG
+ fprintf(stderr,"vcf: %s\nref: %s\n",ptr_vcf,ptr_ref);
+#endif
+
+ int i = 0;
+ while ( ptr_vcf[i] && ptr_vcf[i]==ptr_ref[i] ) i++;
+ if ( ptr_vcf[i] ) return 0; // the deleted sequence cannot be replaced
+ }
+
+ return 1;
+}
+
static inline int splice_csq_del(args_t *args, splice_t *splice, uint32_t ex_beg, uint32_t ex_end)
{
+ if ( splice->check_start )
+ {
+ // check for synonymous start
+ // test/csq/ENST00000375992/incorrect-synon-del-not-start-lost.txt
+ // test/csq/ENST00000368801.2/start-lost.txt
+ // test/csq/ENST00000318249.2/synonymous-start-lost.txt
+ int is_synonymous = shifted_del_synonymous(args, splice, ex_beg, ex_end);
+ if ( is_synonymous )
+ {
+ splice->csq |= CSQ_START_RETAINED;
+ return SPLICE_OVERLAP;
+ }
+ }
+
// coordinates that matter for consequences, eg AC>ACG trimmed to C>CG
splice->ref_beg = splice->vcf.pos + splice->tbeg - 1; // 1b before the deleted base
splice->ref_end = splice->vcf.pos + splice->vcf.rlen - splice->tend - 1; // the last deleted base
#if XDBG
-fprintf(stderr,"del: %s>%s .. ex=%d,%d beg,end=%d,%d tbeg,tend=%d,%d check_utr=%d start,stop,beg,end=%d,%d,%d,%d\n", splice->vcf.ref,splice->vcf.alt,ex_beg,ex_end,splice->ref_beg,splice->ref_end,splice->tbeg,splice->tend,splice->check_utr,splice->check_start,splice->check_stop,splice->check_region_beg,splice->check_region_end);
+fprintf(stderr,"splice_csq_del: %s>%s .. ex=%d,%d beg,end=%d,%d tbeg,tend=%d,%d check_utr=%d start,stop,beg,end=%d,%d,%d,%d\n", splice->vcf.ref,splice->vcf.alt,ex_beg,ex_end,splice->ref_beg,splice->ref_end,splice->tbeg,splice->tend,splice->check_utr,splice->check_start,splice->check_stop,splice->check_region_beg,splice->check_region_end);
#endif
if ( splice->ref_beg + 1 < ex_beg ) // the part before the exon; ref_beg is off by -1
regitr_t *itr = regitr_init(NULL);
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,splice->ref_beg,ex_beg-1, itr) ) // adjacent utr
- csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
regitr_destroy(itr);
}
if ( !csq )
regitr_t *itr = regitr_init(NULL);
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,ex_end+1,splice->ref_end, itr) ) // adjacent utr
- csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
regitr_destroy(itr);
}
if ( !csq )
csq_stage_splice(args, splice->vcf.rec, splice->tr, splice->csq);
return SPLICE_OUTSIDE;
}
-
if ( splice->ref_beg < ex_beg + 2 ) // ref_beg is off by -1
{
if ( splice->check_region_beg ) splice->csq |= CSQ_SPLICE_REGION;
regitr_t *itr = regitr_init(NULL);
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,splice->ref_beg,ex_beg-1, itr) ) // adjacent utr
- csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
regitr_destroy(itr);
}
if ( !csq )
regitr_t *itr = regitr_init(NULL);
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,ex_end+1,splice->ref_end, itr) ) // adjacent utr
- csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
regitr_destroy(itr);
}
if ( !csq )
}
static inline int splice_csq(args_t *args, splice_t *splice, uint32_t ex_beg, uint32_t ex_end)
{
- splice->csq = 0;
splice->vcf.alen = strlen(splice->vcf.alt);
int rlen1 = splice->vcf.rlen - 1, alen1 = splice->vcf.alen - 1, i = 0;
return 0;
}
+
// return value: 0 added, 1 overlapping variant, 2 silent discard (intronic,alt=ref)
int hap_init(args_t *args, hap_node_t *parent, hap_node_t *child, gf_cds_t *cds, bcf1_t *rec, int ial)
{
if ( child->icds!=tr->ncds-1 ) splice.check_region_end = 1;
#if XDBG
-fprintf(stderr,"\n%d [%s][%s] check start:%d,stop:%d\n",splice.vcf.pos+1,splice.vcf.ref,splice.vcf.alt,splice.check_start,splice.check_stop);
+fprintf(stderr,"\nhap_init: %d [%s][%s] check start:%d,stop:%d\n",splice.vcf.pos+1,splice.vcf.ref,splice.vcf.alt,splice.check_start,splice.check_stop);
#endif
int ret = splice_csq(args, &splice, cds->beg, cds->beg + cds->len - 1);
#if XDBG
#endif
if ( ret==SPLICE_VAR_REF ) return 2; // not a variant, eg REF=CA ALT=CA
- if ( ret==SPLICE_OUTSIDE || ret==SPLICE_OVERLAP ) // not a coding csq
+ if ( ret==SPLICE_OUTSIDE || ret==SPLICE_OVERLAP || splice.csq==CSQ_START_LOST ) // not a coding csq
{
free(splice.kref.s);
free(splice.kalt.s);
if ( len < 0 ) // overlapping variants
{
free(str.s);
+ free(splice.kref.s);
+ free(splice.kalt.s);
return 1;
}
kputsn_(tr->ref + N_REF_PAD + parent->rbeg + parent->rlen - tr->beg, len, &str);
if ( !child->csq ) child->csq |= CSQ_CODING_SEQUENCE; // hack, specifically for ENST00000390520/deletion-overlap.vcf
}
+
free(splice.kref.s);
free(splice.kalt.s);
return 0;
void cds_translate(kstring_t *_ref, kstring_t *_seq, uint32_t sbeg, uint32_t rbeg, uint32_t rend, int strand, kstring_t *tseq, int fill)
{
#if XDBG
-fprintf(stderr,"translate: %d %d %d fill=%d seq.l=%d\n",sbeg,rbeg,rend,fill,(int)_seq->l);
+fprintf(stderr,"\ntranslate: %d %d %d fill=%d seq.l=%d\n",sbeg,rbeg,rend,fill,(int)_seq->l);
#endif
char tmp[3], *codon, *end;
int i, len, npad;
#if DBG>1
fprintf(stderr," npad: %d\n",npad);
#endif
-if ( !(npad>=0 && sbeg+seq.l+npad<=seq.m) ) fprintf(stderr,"sbeg=%d seq.l=%d seq.m=%d\n",sbeg,(int)seq.l,(int)seq.m);
+ if ( !(npad>=0 && sbeg+seq.l+npad<=seq.m) ) fprintf(stderr,"sbeg=%d seq.l=%d seq.m=%d npad=%d\n",sbeg,(int)seq.l,(int)seq.m,npad);
assert( npad>=0 && sbeg+seq.l+npad<=seq.m ); // todo: first codon on the rev strand
if ( npad==2 )
for (; i>=0 && end>seq.s; i--) tmp[i] = *(--end);
#if DBG>1
fprintf(stderr,"\t i=%d\n", i);
- if(i==1)fprintf(stderr,"[0] %c\n",tmp[2]);
- if(i==0)fprintf(stderr,"[0] %c%c\n",tmp[1],tmp[2]);
+ if(i==1)fprintf(stderr,"[0] %c\n",tmp[2]);
+ if(i==0)fprintf(stderr,"[0] %c%c\n",tmp[1],tmp[2]);
#endif
if ( i==-1 )
{
kputs(csq->vstr.s, str);
}
+void kprint_aa_prediction(args_t *args, int beg, kstring_t *aa, kstring_t *str)
+{
+ if ( !args->brief_predictions )
+ kputs(aa->s, str);
+ else
+ {
+ int len = aa->l;
+ if ( aa->s[len-1]=='*' ) len--;
+ kputc(aa->s[0], str);
+ kputs("..", str);
+ kputw(beg+len, str);
+ }
+}
+
void hap_add_csq(args_t *args, hap_t *hap, hap_node_t *node, int tlen, int ibeg, int iend, int dlen, int indel)
{
int i;
tscript_t *tr = hap->tr;
int ref_node = tr->strand==STRAND_FWD ? ibeg : iend;
-
int icsq = node->ncsq_list++;
hts_expand0(csq_t,node->ncsq_list,node->mcsq_list,node->csq_list);
csq_t *csq = &node->csq_list[icsq];
int aa_sbeg = tr->strand==STRAND_FWD ? node2sbeg(ibeg)/3+1 : (tlen - node2send(iend))/3+1;
kputc_('|', &str);
kputw(aa_rbeg, &str);
- kputs(hap->tref.s, &str);
+ kprint_aa_prediction(args,aa_rbeg,&hap->tref,&str);
if ( !(csq->type.type & CSQ_SYNONYMOUS_VARIANT) )
{
kputc_('>', &str);
kputw(aa_sbeg, &str);
- kputs(hap->tseq.s, &str);
+ kprint_aa_prediction(args,aa_sbeg,&hap->tseq,&str);
}
kputc_('|', &str);
int icsq = 2*csq->idx + ihap;
if ( icsq >= args->ncsq_max ) // more than ncsq_max consequences, so can't fit it in FMT
{
- int print_warning = 1;
- if ( args->quiet )
+ if ( args->verbosity && (!args->ncsq_small_warned || args->verbosity > 1) )
{
- if ( args->quiet > 1 || args->ncsq_small_warned ) print_warning = 0;
+ fprintf(stderr,
+ "Warning: Too many consequences for sample %s at %s:%"PRId64", keeping the first %d and skipping the rest.\n",
+ args->hdr->samples[ismpl],bcf_hdr_id2name(args->hdr,args->rid),(int64_t) vrec->line->pos+1,csq->idx);
+ if ( !args->ncsq_small_warned )
+ fprintf(stderr," The limit can be increased by setting the --ncsq parameter. This warning is printed only once.\n");
args->ncsq_small_warned = 1;
}
- if ( print_warning )
- {
- fprintf(stderr,"Warning: --ncsq %d is too small to annotate %s at %s:%d with %d-th csq\n",
- args->ncsq_max/2,args->hdr->samples[ismpl],bcf_hdr_id2name(args->hdr,args->rid),vrec->line->pos+1,csq->idx+1);
- if ( args->quiet ) fprintf(stderr,"(This warning is printed only once)\n");
- }
break;
}
if ( vrec->nfmt < 1 + icsq/32 ) vrec->nfmt = 1 + icsq/32;
{
int i,j;
tr_heap_t *heap = args->active_tr;
-
while ( heap->ndat && heap->dat[0]->end<=pos )
{
tscript_t *tr = heap->dat[0];
khp_delete(trhp, heap);
-
args->hap->tr = tr;
if ( tr->root && tr->root->nchild ) // normal, non-localized calling
{
#define SWAP(type_t, a, b) { type_t t = a; a = b; b = t; }
-void vbuf_push(args_t *args, bcf1_t **rec_ptr)
+vbuf_t *vbuf_push(args_t *args, bcf1_t **rec_ptr)
{
int i;
i = rbuf_append(&args->vcf_rbuf);
if ( !args->vcf_buf[i] ) args->vcf_buf[i] = (vbuf_t*) calloc(1,sizeof(vbuf_t));
args->vcf_buf[i]->n = 0;
+ args->vcf_buf[i]->keep_until = 0;
}
vbuf_t *vbuf = args->vcf_buf[i];
vbuf->n++;
int ret;
khint_t k = kh_put(pos2vbuf, args->pos2vbuf, (int)rec->pos, &ret);
kh_val(args->pos2vbuf,k) = vbuf;
+
+ return vbuf;
}
-void vbuf_flush(args_t *args)
+void vbuf_flush(args_t *args, uint32_t pos)
{
- if ( args->active_tr->ndat ) return; // cannot output buffered VCF lines (args.vbuf) until all active transcripts are gone
-
int i,j;
- while ( (i=rbuf_shift(&args->vcf_rbuf))>=0 )
+ while ( args->vcf_rbuf.n )
{
- vbuf_t *vbuf = args->vcf_buf[i];
+ vbuf_t *vbuf;
+ if ( !args->local_csq && args->active_tr->ndat )
+ {
+ // check if the first active transcript starts beyond the first buffered VCF record,
+ // cannot output buffered VCF lines (args.vbuf) until the active transcripts are gone
+ vbuf = args->vcf_buf[ args->vcf_rbuf.f ];
+ if ( vbuf->keep_until > pos ) break;
+ assert( vbuf->n );
+ }
+
+ i = rbuf_shift(&args->vcf_rbuf);
+ assert( i>=0 );
+ vbuf = args->vcf_buf[i];
+ int pos = vbuf->n ? vbuf->vrec[0]->line->pos : -1;
for (i=0; i<vbuf->n; i++)
{
vrec_t *vrec = vbuf->vrec[i];
}
if ( !vrec->nvcsq )
{
- bcf_write(args->out_fh, args->hdr, vrec->line);
+ if ( bcf_write(args->out_fh, args->hdr, vrec->line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname?args->output_fname:"standard output");
+ int save_pos = vrec->line->pos;
+ bcf_empty(vrec->line);
+ vrec->line->pos = save_pos; // this is necessary for compound variants
continue;
}
if ( args->hdr_nsmpl )
{
if ( vrec->nfmt < args->nfmt_bcsq )
- for (j=1; j<args->hdr_nsmpl; j++) memcpy(vrec->smpl+j*vrec->nfmt, vrec->smpl+j*args->nfmt_bcsq, vrec->nfmt*sizeof(*vrec->smpl));
+ for (j=1; j<args->hdr_nsmpl; j++)
+ memmove(&vrec->smpl[j*vrec->nfmt], &vrec->smpl[j*args->nfmt_bcsq], vrec->nfmt*sizeof(*vrec->smpl));
bcf_update_format_int32(args->hdr, vrec->line, args->bcsq_tag, vrec->smpl, args->hdr_nsmpl*vrec->nfmt);
}
vrec->nvcsq = 0;
- bcf_write(args->out_fh, args->hdr, vrec->line);
+ if ( bcf_write(args->out_fh, args->hdr, vrec->line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname?args->output_fname:"standard output");
+ int save_pos = vrec->line->pos;
+ bcf_empty(vrec->line);
+ vrec->line->pos = save_pos;
}
- if ( vbuf->n )
+ if ( pos!=-1 )
{
- khint_t k = kh_get(pos2vbuf, args->pos2vbuf, vbuf->vrec[0]->line->pos);
+ khint_t k = kh_get(pos2vbuf, args->pos2vbuf, pos);
if ( k != kh_end(args->pos2vbuf) ) kh_del(pos2vbuf, args->pos2vbuf, k);
}
vbuf->n = 0;
}
+ if ( args->active_tr->ndat ) return;
for (i=0; i<args->nrm_tr; i++)
{
int pad_end = len - (tr->end - tr->beg + 1 + pad_beg);
if ( pad_beg + pad_end != 2*N_REF_PAD )
{
- char *ref = (char*) malloc(tr->end - tr->beg + 1 + 2*N_REF_PAD);
+ char *ref = (char*) malloc(tr->end - tr->beg + 1 + 2*N_REF_PAD + 1);
for (i=0; i < N_REF_PAD - pad_beg; i++) ref[i] = 'N';
memcpy(ref+i, tr->ref, len);
+ len += i;
for (i=0; i < N_REF_PAD - pad_end; i++) ref[i+len] = 'N';
+ ref[i+len] = 0;
free(tr->ref);
tr->ref = ref;
}
static void sanity_check_ref(args_t *args, tscript_t *tr, bcf1_t *rec)
{
- char *ref = tr->ref + (rec->pos + N_REF_PAD >= tr->beg ? rec->pos - tr->beg + N_REF_PAD : 0);
- char *vcf = rec->d.allele[0] + (rec->pos + N_REF_PAD >= tr->beg ? 0 : tr->beg - N_REF_PAD - rec->pos);
- assert( vcf - rec->d.allele[0] < strlen(rec->d.allele[0]) );
- while ( *ref && *vcf )
+ int vbeg = 0;
+ int rbeg = rec->pos - tr->beg + N_REF_PAD;
+ if ( rbeg < 0 ) { vbeg += abs(rbeg); rbeg = 0; }
+ char *ref = tr->ref + rbeg;
+ char *vcf = rec->d.allele[0] + vbeg;
+ assert( vcf - rec->d.allele[0] < strlen(rec->d.allele[0]) && ref - tr->ref < tr->end - tr->beg + 2*N_REF_PAD );
+ int i = 0;
+ while ( ref[i] && vcf[i] )
{
- if ( *ref!=*vcf && toupper(*ref)!=toupper(*vcf) )
- error("Error: the fasta reference does not match the VCF REF allele at %s:%d .. %s\n", bcf_seqname(args->hdr,rec),rec->pos+1,rec->d.allele[0]);
- ref++;
- vcf++;
+ if ( ref[i]!=vcf[i] && toupper(ref[i])!=toupper(vcf[i]) )
+ error("Error: the fasta reference does not match the VCF REF allele at %s:%"PRId64" .. fasta=%c vcf=%c\n",
+ bcf_seqname(args->hdr,rec),(int64_t) rec->pos+vbeg+1,ref[i],vcf[i]);
+ i++;
}
}
for (i=1; i<rec->n_allele; i++)
{
+ if ( rec->d.allele[i][0]=='<' || rec->d.allele[i][0]=='*' ) { continue; }
if ( hap_init(args, &root, &node, cds, rec, i)!=0 ) continue;
csq_t csq;
int aa_sbeg = tr->strand==STRAND_FWD ? node.sbeg/3+1 : (tr->nsref - 2*N_REF_PAD + node.dlen - node.sbeg - alen)/3+1;
kputc_('|', &str);
kputw(aa_rbeg, &str);
- kputs(tref->s, &str);
+ kprint_aa_prediction(args,aa_rbeg,tref,&str);
if ( !(csq_type & CSQ_SYNONYMOUS_VARIANT) )
{
kputc_('>', &str);
kputw(aa_sbeg, &str);
- kputs(tseq->s, &str);
+ kprint_aa_prediction(args,aa_sbeg,tseq,&str);
}
kputc_('|', &str);
kputw(rec->pos+1, &str);
return ret;
}
-int test_cds(args_t *args, bcf1_t *rec)
+int test_cds(args_t *args, bcf1_t *rec, vbuf_t *vbuf)
{
+ static int overlaps_warned = 0, multiploid_warned = 0;
+
int i, ret = 0, hap_ret;
const char *chr = bcf_seqname(args->hdr,rec);
// note that the off-by-one extension of rlen is deliberate to account for insertions
gf_cds_t *cds = regitr_payload(args->itr,gf_cds_t*);
tscript_t *tr = cds->tr;
if ( !GF_is_coding(tr->type) ) continue;
+ if ( vbuf->keep_until < tr->end ) vbuf->keep_until = tr->end;
ret = 1;
if ( !tr->root )
{
// overlapping or intron variant, cannot apply
if ( hap_ret==1 )
{
- if ( !args->quiet )
- fprintf(stderr,"Warning: Skipping overlapping variants at %s:%d\t%s>%s\n", chr,rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ if ( args->verbosity && (!overlaps_warned || args->verbosity > 1) )
+ {
+ fprintf(stderr,
+ "Warning: Skipping overlapping variants at %s:%"PRId64"\t%s>%s.\n",
+ chr,(int64_t) rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ if ( !overlaps_warned )
+ fprintf(stderr," This message is printed only once, the verbosity can be increased with `--verbose 2`\n");
+ overlaps_warned = 1;
+ }
if ( args->out )
- fprintf(args->out,"LOG\tWarning: Skipping overlapping variants at %s:%d\t%s>%s\n", chr,rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ fprintf(args->out,"LOG\tWarning: Skipping overlapping variants at %s:%"PRId64"\t%s>%s\n", chr,(int64_t) rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
}
else ret = 1; // prevent reporting as intron in test_tscript
hap_destroy(child);
ngts /= bcf_hdr_nsamples(args->hdr);
if ( ngts!=1 && ngts!=2 )
{
- if ( !args->quiet )
- fprintf(stderr,"Warning: Skipping site with non-diploid/non-haploid genotypes at %s:%d\t%s>%s\n", chr,rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ if ( args->verbosity && (!multiploid_warned || args->verbosity > 1) )
+ {
+ fprintf(stderr,
+ "Warning: Skipping site with non-diploid/non-haploid genotypes at %s:%"PRId64"\t%s>%s.\n",
+ chr,(int64_t) rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ if ( !multiploid_warned )
+ fprintf(stderr," This message is printed only once, the verbosity can be increased with `--verbose 2`\n");
+ multiploid_warned = 1;
+ }
if ( args->out )
- fprintf(args->out,"LOG\tWarning: Skipping site with non-diploid/non-haploid genotypes at %s:%d\t%s>%s\n", chr,rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ fprintf(args->out,"LOG\tWarning: Skipping site with non-diploid/non-haploid genotypes at %s:%"PRId64"\t%s>%s\n", chr,(int64_t) rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
continue;
}
for (ismpl=0; ismpl<args->smpl->n; ismpl++)
if ( !bcf_gt_is_phased(gt[0]) && !bcf_gt_is_phased(gt[1]) )
{
if ( args->phase==PHASE_REQUIRE )
- error("Unphased heterozygous genotype at %s:%d, sample %s. See the --phase option.\n", chr,rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]]);
+ error("Unphased heterozygous genotype at %s:%"PRId64", sample %s. See the --phase option.\n", chr,(int64_t) rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]]);
if ( args->phase==PHASE_SKIP )
continue;
if ( args->phase==PHASE_NON_REF )
// overlapping or intron variant, cannot apply
if ( hap_ret==1 )
{
- if ( !args->quiet )
- fprintf(stderr,"Warning: Skipping overlapping variants at %s:%d, sample %s\t%s>%s\n",
- chr,rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]],rec->d.allele[0],rec->d.allele[ial]);
+ if ( args->verbosity && (!overlaps_warned || args->verbosity > 1) )
+ {
+ fprintf(stderr,
+ "Warning: Skipping overlapping variants at %s:%"PRId64", sample %s\t%s>%s.\n",
+ chr,(int64_t) rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]],rec->d.allele[0],rec->d.allele[ial]);
+ if ( !overlaps_warned )
+ fprintf(stderr," This message is printed only once, the verbosity can be increased with `--verbose 2`\n");
+ overlaps_warned = 1;
+ }
if ( args->out )
- fprintf(args->out,"LOG\tWarning: Skipping overlapping variants at %s:%d, sample %s\t%s>%s\n",
- chr,rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]],rec->d.allele[0],rec->d.allele[ial]);
+ fprintf(args->out,"LOG\tWarning: Skipping overlapping variants at %s:%"PRId64", sample %s\t%s>%s\n",
+ chr,(int64_t) rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]],rec->d.allele[0],rec->d.allele[ial]);
}
hap_destroy(child);
continue;
if ( icsq >= args->ncsq_max ) // more than ncsq_max consequences, so can't fit it in FMT
{
int ismpl = args->smpl->idx[i];
- int print_warning = 1;
- if ( args->quiet )
+ if ( args->verbosity && (!args->ncsq_small_warned || args->verbosity > 1) )
{
- if ( args->quiet > 1 || args->ncsq_small_warned ) print_warning = 0;
+ fprintf(stderr,
+ "Warning: Too many consequences for sample %s at %s:%"PRId64", keeping the first %d and skipping the rest.\n",
+ args->hdr->samples[ismpl],bcf_hdr_id2name(args->hdr,args->rid),(int64_t) vrec->line->pos+1,icsq+1);
+ if ( !args->ncsq_small_warned )
+ fprintf(stderr," The limit can be increased by setting the --ncsq parameter. This warning is printed only once.\n");
args->ncsq_small_warned = 1;
}
- if ( print_warning )
- {
- fprintf(stderr,"Warning: --ncsq %d is too small to annotate %s at %s:%d with %d-th csq\n",
- args->ncsq_max/2,args->hdr->samples[ismpl],bcf_hdr_id2name(args->hdr,args->rid),vrec->line->pos+1,csq->idx+1);
- if ( args->quiet ) fprintf(stderr,"(This warning is printed only once)\n");
- }
- break;
}
if ( vrec->nfmt < 1 + icsq/32 ) vrec->nfmt = 1 + icsq/32;
vrec->smpl[i*args->nfmt_bcsq + icsq/32] |= 1 << (icsq % 32);
tscript_t *tr = splice.tr = utr->tr;
for (i=1; i<rec->n_allele; i++)
{
- if ( rec->d.allele[1][0]=='<' || rec->d.allele[1][0]=='*' ) { continue; }
+ if ( rec->d.allele[i][0]=='<' || rec->d.allele[i][0]=='*' ) { continue; }
splice.vcf.alt = rec->d.allele[i];
+ splice.csq = 0;
int splice_ret = splice_csq(args, &splice, utr->beg, utr->end);
if ( splice_ret!=SPLICE_INSIDE && splice_ret!=SPLICE_OVERLAP ) continue;
csq_t csq;
{
if ( rec->d.allele[1][0]=='<' || rec->d.allele[1][0]=='*' ) { continue; }
splice.vcf.alt = rec->d.allele[i];
+ splice.csq = 0;
splice_csq(args, &splice, exon->beg, exon->end);
if ( splice.csq ) ret = 1;
}
tscript_t *tr = splice.tr = regitr_payload(args->itr, tscript_t*);
for (i=1; i<rec->n_allele; i++)
{
- if ( rec->d.allele[1][0]=='<' || rec->d.allele[1][0]=='*' ) { continue; }
+ if ( rec->d.allele[i][0]=='<' || rec->d.allele[i][0]=='*' ) { continue; }
splice.vcf.alt = rec->d.allele[i];
+ splice.csq = 0;
int splice_ret = splice_csq(args, &splice, tr->beg, tr->end);
if ( splice_ret!=SPLICE_INSIDE && splice_ret!=SPLICE_OVERLAP ) continue; // SPLICE_OUTSIDE or SPLICE_REF
csq_t csq;
return ret;
}
-void process(args_t *args, bcf1_t **rec_ptr)
+void test_symbolic_alt(args_t *args, bcf1_t *rec)
+{
+ static int warned = 0;
+ if ( args->verbosity && (!warned && args->verbosity > 0) )
+ {
+ fprintf(stderr,"Warning: The support for symbolic ALT insertions is experimental.\n");
+ warned = 1;
+ }
+
+ const char *chr = bcf_seqname(args->hdr,rec);
+
+ // only insertions atm
+ int beg = rec->pos + 1;
+ int end = beg;
+ int csq_class = CSQ_ELONGATION;
+
+ int hit = 0;
+ if ( regidx_overlap(args->idx_cds,chr,beg,end, args->itr) )
+ {
+ while ( regitr_overlap(args->itr) )
+ {
+ csq_t csq;
+ memset(&csq, 0, sizeof(csq_t));
+ gf_cds_t *cds = regitr_payload(args->itr,gf_cds_t*);
+ tscript_t *tr = cds->tr;
+ csq.type.type = (GF_is_coding(tr->type) ? CSQ_CODING_SEQUENCE : CSQ_NON_CODING) | csq_class;
+ csq.pos = rec->pos;
+ csq.type.biotype = tr->type;
+ csq.type.strand = tr->strand;
+ csq.type.trid = tr->id;
+ csq.type.gene = tr->gene->name;
+ csq_stage(args, &csq, rec);
+ hit = 1;
+ }
+ }
+ if ( regidx_overlap(args->idx_utr,chr,beg,end, args->itr) )
+ {
+ while ( regitr_overlap(args->itr) )
+ {
+ csq_t csq;
+ memset(&csq, 0, sizeof(csq_t));
+ gf_utr_t *utr = regitr_payload(args->itr, gf_utr_t*);
+ tscript_t *tr = utr->tr;
+ csq.type.type = (utr->which==prime5 ? CSQ_UTR5 : CSQ_UTR3) | csq_class;
+ csq.pos = rec->pos;
+ csq.type.biotype = tr->type;
+ csq.type.strand = tr->strand;
+ csq.type.trid = tr->id;
+ csq.type.gene = tr->gene->name;
+ csq_stage(args, &csq, rec);
+ hit = 1;
+ }
+ }
+ if ( regidx_overlap(args->idx_exon,chr,beg,end, args->itr) )
+ {
+ splice_t splice;
+ splice_init(&splice, rec);
+ splice.check_acceptor = splice.check_donor = 1;
+
+ while ( regitr_overlap(args->itr) )
+ {
+ gf_exon_t *exon = regitr_payload(args->itr, gf_exon_t*);
+ splice.tr = exon->tr;
+ if ( !splice.tr->ncds ) continue; // not a coding transcript, no interest in splice sites
+ splice.check_region_beg = splice.tr->beg==exon->beg ? 0 : 1;
+ splice.check_region_end = splice.tr->end==exon->end ? 0 : 1;
+ splice.vcf.alt = rec->d.allele[1];
+ splice.csq = csq_class;
+ splice_csq(args, &splice, exon->beg, exon->end);
+ if ( splice.csq ) hit = 1;
+ }
+ }
+ if ( !hit && regidx_overlap(args->idx_tscript,chr,beg,end, args->itr) )
+ {
+ splice_t splice;
+ splice_init(&splice, rec);
+
+ while ( regitr_overlap(args->itr) )
+ {
+ csq_t csq;
+ memset(&csq, 0, sizeof(csq_t));
+ tscript_t *tr = splice.tr = regitr_payload(args->itr, tscript_t*);
+ splice.vcf.alt = rec->d.allele[1];
+ splice.csq = csq_class;
+ int splice_ret = splice_csq(args, &splice, tr->beg, tr->end);
+ if ( splice_ret!=SPLICE_INSIDE && splice_ret!=SPLICE_OVERLAP ) continue; // SPLICE_OUTSIDE or SPLICE_REF
+ csq.type.type = (GF_is_coding(tr->type) ? CSQ_INTRON : CSQ_NON_CODING) | csq_class;
+ csq.pos = rec->pos;
+ csq.type.biotype = tr->type;
+ csq.type.strand = tr->strand;
+ csq.type.trid = tr->id;
+ csq.type.gene = tr->gene->name;
+ csq_stage(args, &csq, rec);
+ }
+ }
+}
+
+void debug_print_buffers(args_t *args, int pos)
+{
+ int i,j;
+ fprintf(stderr,"debug_print_buffers at %d\n", pos);
+ fprintf(stderr,"vbufs:\n");
+ for (i=0; i<args->vcf_rbuf.n; i++)
+ {
+ int k = rbuf_kth(&args->vcf_rbuf, i);
+ vbuf_t *vbuf = args->vcf_buf[k];
+
+ fprintf(stderr,"\tvbuf %d:\n", i);
+ for (j=0; j<vbuf->n; j++)
+ {
+ vrec_t *vrec = vbuf->vrec[j];
+ fprintf(stderr,"\t\t%"PRId64" .. nvcsq=%d\n", (int64_t) vrec->line->pos+1, vrec->nvcsq);
+ }
+ }
+ fprintf(stderr,"pos2vbuf:");
+ khint_t k;
+ for (k = 0; k < kh_end(args->pos2vbuf); ++k)
+ if (kh_exist(args->pos2vbuf, k)) fprintf(stderr," %d",1+(int)kh_key(args->pos2vbuf, k));
+ fprintf(stderr,"\n");
+ fprintf(stderr,"active_tr: %d\n", args->active_tr->ndat);
+}
+
+static void process(args_t *args, bcf1_t **rec_ptr)
{
if ( !rec_ptr )
{
hap_flush(args, REGIDX_MAX);
- vbuf_flush(args);
+ vbuf_flush(args, REGIDX_MAX);
return;
}
bcf1_t *rec = *rec_ptr;
+ static int32_t prev_rid = -1, prev_pos = -1;
+ if ( prev_rid!=rec->rid ) { prev_rid = rec->rid; prev_pos = rec->pos; }
+ if ( prev_pos > rec->pos )
+ error("Error: The file is not sorted, %s:%d comes before %s:%"PRId64"\n",bcf_seqname(args->hdr,rec),prev_pos+1,bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
int call_csq = 1;
- if ( !rec->n_allele ) call_csq = 0; // no alternate allele
- else if ( rec->n_allele==2 && (rec->d.allele[1][0]=='<' || rec->d.allele[1][0]=='*') ) call_csq = 0; // gVCF, no alt allele
- else if ( rec->d.allele[1][0]=='<' && rec->d.allele[1][0]!='*') call_csq = 0; // a symbolic allele, not ready for CNVs etc
- else if ( args->filter )
+ if ( rec->n_allele < 2 ) call_csq = 0; // no alternate allele
+ else if ( rec->n_allele==2 && (rec->d.allele[1][0]=='*' || rec->d.allele[1][1]=='*') ) call_csq = 0; // gVCF, not an alt allele
+ else if ( rec->d.allele[1][0]=='<' )
+ {
+ if ( strncmp("<INS",rec->d.allele[1], 4) ) call_csq = 0; // only <INS[:.*]> is supported at the moment
+ }
+ if ( call_csq && args->filter )
{
call_csq = filter_test(args->filter, rec, NULL);
if ( args->filter_logic==FLT_EXCLUDE ) call_csq = call_csq ? 0 : 1;
{
if ( !args->out_fh ) return; // not a VCF output
vbuf_push(args, rec_ptr);
- vbuf_flush(args);
+ hap_flush(args, rec->pos-1);
+ vbuf_flush(args, rec->pos-1);
return;
}
if ( args->rid != rec->rid )
{
hap_flush(args, REGIDX_MAX);
- vbuf_flush(args);
+ vbuf_flush(args, REGIDX_MAX);
}
args->rid = rec->rid;
- vbuf_push(args, rec_ptr);
+ vbuf_t *vbuf = vbuf_push(args, rec_ptr);
- int hit = args->local_csq ? test_cds_local(args, rec) : test_cds(args, rec);
- hit += test_utr(args, rec);
- hit += test_splice(args, rec);
- if ( !hit ) test_tscript(args, rec);
+ if ( rec->d.allele[1][0]!='<' )
+ {
+ int hit = args->local_csq ? test_cds_local(args, rec) : test_cds(args, rec, vbuf);
+ hit += test_utr(args, rec);
+ hit += test_splice(args, rec);
+ if ( !hit ) test_tscript(args, rec);
+ }
+ else
+ test_symbolic_alt(args, rec);
- hap_flush(args, rec->pos-1);
- vbuf_flush(args);
+ if ( rec->pos > 0 )
+ {
+ hap_flush(args, rec->pos-1);
+ vbuf_flush(args, rec->pos-1);
+ }
return;
}
" -g, --gff-annot <file> gff3 annotation file\n"
"\n"
"CSQ options:\n"
+ " -b, --brief-predictions annotate with abbreviated protein-changing predictions\n"
" -c, --custom-tag <string> use this tag instead of the default BCSQ\n"
" -l, --local-csq localized predictions, consider only one VCF record at a time\n"
" -n, --ncsq <int> maximum number of consequences to consider per site [16]\n"
" -e, --exclude <expr> exclude sites for which the expression is true\n"
" --force run even if some sanity checks fail\n"
" -i, --include <expr> select sites for which the expression is true\n"
+ " --no-version do not append version and command line to the header\n"
" -o, --output <file> write output to a file [standard output]\n"
" -O, --output-type <b|u|z|v|t> b: compressed BCF, u: uncompressed BCF, z: compressed VCF\n"
" v: uncompressed VCF, t: plain tab-delimited text output [v]\n"
- " -q, --quiet suppress warning messages. Can be given two times for even less messages\n"
" -r, --regions <region> restrict to comma-separated list of regions\n"
" -R, --regions-file <file> restrict to regions listed in a file\n"
" -s, --samples <-|list> samples to include or \"-\" to apply all variants and ignore samples\n"
" -S, --samples-file <file> samples to include\n"
" -t, --targets <region> similar to -r but streams rather than index-jumps\n"
" -T, --targets-file <file> similar to -R but streams rather than index-jumps\n"
+ " --threads <int> use multithreading with <int> worker threads [0]\n"
+ " -v, --verbose <int> verbosity level 0-2 [1]\n"
"\n"
"Example:\n"
" bcftools csq -f hs37d5.fa -g Homo_sapiens.GRCh37.82.gff3.gz in.vcf\n"
args->output_type = FT_VCF;
args->bcsq_tag = "BCSQ";
args->ncsq_max = 2*16;
+ args->verbosity = 1;
+ args->record_cmd_line = 1;
static struct option loptions[] =
{
{"force",0,0,1},
+ {"threads",required_argument,NULL,2},
{"help",0,0,'h'},
{"ncsq",1,0,'n'},
+ {"brief-predictions",0,0,'b'},
{"custom-tag",1,0,'c'},
{"local-csq",0,0,'l'},
{"gff-annot",1,0,'g'},
{"output-type",1,NULL,'O'},
{"phase",1,0,'p'},
{"quiet",0,0,'q'},
+ {"verbose",1,0,'v'},
{"regions",1,0,'r'},
{"regions-file",1,0,'R'},
{"samples",1,0,'s'},
{"samples-file",1,0,'S'},
{"targets",1,0,'t'},
{"targets-file",1,0,'T'},
+ {"no-version",no_argument,NULL,3},
{0,0,0,0}
};
int c, targets_is_file = 0, regions_is_file = 0;
- char *targets_list = NULL, *regions_list = NULL;
- while ((c = getopt_long(argc, argv, "?hr:R:t:T:i:e:f:o:O:g:s:S:p:qc:ln:",loptions,NULL)) >= 0)
+ char *targets_list = NULL, *regions_list = NULL, *tmp;
+ while ((c = getopt_long(argc, argv, "?hr:R:t:T:i:e:f:o:O:g:s:S:p:qc:ln:bv:",loptions,NULL)) >= 0)
{
switch (c)
{
case 1 : args->force = 1; break;
+ case 2 :
+ args->n_threads = strtol(optarg,&tmp,10);
+ if ( *tmp ) error("Could not parse argument: --threads %s\n", optarg);
+ break;
+ case 3 : args->record_cmd_line = 0; break;
+ case 'b': args->brief_predictions = 1; break;
case 'l': args->local_csq = 1; break;
case 'c': args->bcsq_tag = optarg; break;
- case 'q': args->quiet++; break;
+ case 'q': error("Error: the -q option has been deprecated, use -v, --verbose instead.\n"); break;
+ case 'v':
+ args->verbosity = atoi(optarg);
+ if ( args->verbosity<0 || args->verbosity>2 ) error("Error: expected integer 0-2 with -v, --verbose\n");
+ break;
case 'p':
switch (optarg[0])
{
error("Failed to read the targets: %s\n", targets_list);
if ( regions_list && bcf_sr_set_regions(args->sr, regions_list, regions_is_file)<0 )
error("Failed to read the regions: %s\n", regions_list);
+ if ( bcf_sr_set_threads(args->sr, args->n_threads)<0 ) error("Failed to create %d extra threads\n", args->n_threads);
if ( !bcf_sr_add_reader(args->sr, fname) )
- error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->sr->errnum));
+ error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->sr->errnum));
args->hdr = bcf_sr_get_header(args->sr,0);
init_data(args);
destroy_data(args);
bcf_sr_destroy(args->sr);
free(args);
-
return 0;
}
#include "bcftools.pysam.h"
+//$bt csq -f $ref -g $gff -p r -Ou -o /dev/null /lustre/scratch116/vr/projects/g1k/phase3/release/ALL.chr4.phase3_shapeit2_mvncall_integrated_v5a.20130502.genotypes.vcf.gz
+
+
/* The MIT License
Copyright (c) 2016-2018 Genome Research Ltd.
*/
/*
Things that would be nice to have
+ - dynamic N_REF_PAD
- for stop-lost events (also in frameshifts) report the number of truncated aa's
- memory could be greatly reduced by indexing gff (but it is quite compact already)
- deletions that go beyond transcript boundaries are not checked at sequence level
splice_region_variant .. change within 1-3 bases of the exon or 3-8 bases of the intron
synonymous_variant .. DNA sequence variant resulting in no amino acid change
stop_retained_variant .. different stop codon
+ start_retained_variant .. start codon retained by indel realignment
non_coding_variant .. variant in non-coding sequence, such as RNA gene
5_prime_UTR_variant
3_prime_UTR_variant
#include <stdlib.h>
#include <getopt.h>
#include <math.h>
+#include <inttypes.h>
#include <htslib/hts.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/faidx.h>
#include <errno.h>
#include <unistd.h>
-#include <stdint.h>
#include <ctype.h>
#include "bcftools.h"
#include "filter.h"
#define CSQ_UPSTREAM_STOP (1<<19) // adds * in front of the csq string
#define CSQ_INCOMPLETE_CDS (1<<20) // to remove START/STOP in incomplete CDS, see ENSG00000173376/synon.vcf
#define CSQ_CODING_SEQUENCE (1<<21) // cannot tell exactly what it is, but it does affect the coding sequence
+#define CSQ_ELONGATION (1<<22) // symbolic insertion
+#define CSQ_START_RETAINED (1<<23)
// Haplotype-aware consequences, printed in one vcf record only, the rest has a reference @12345
#define CSQ_COMPOUND (CSQ_SYNONYMOUS_VARIANT|CSQ_MISSENSE_VARIANT|CSQ_STOP_LOST|CSQ_STOP_GAINED| \
CSQ_INFRAME_DELETION|CSQ_INFRAME_INSERTION|CSQ_FRAMESHIFT_VARIANT| \
CSQ_START_LOST|CSQ_STOP_RETAINED|CSQ_INFRAME_ALTERING|CSQ_INCOMPLETE_CDS| \
- CSQ_UPSTREAM_STOP)
-#define CSQ_START_STOP (CSQ_STOP_LOST|CSQ_STOP_GAINED|CSQ_STOP_RETAINED|CSQ_START_LOST)
+ CSQ_UPSTREAM_STOP|CSQ_START_RETAINED)
+#define CSQ_START_STOP (CSQ_STOP_LOST|CSQ_STOP_GAINED|CSQ_STOP_RETAINED|CSQ_START_LOST|CSQ_START_RETAINED)
#define CSQ_PRN_STRAND(csq) ((csq)&CSQ_COMPOUND && !((csq)&(CSQ_SPLICE_ACCEPTOR|CSQ_SPLICE_DONOR|CSQ_SPLICE_REGION)))
#define CSQ_PRN_TSCRIPT (~(CSQ_INTRON|CSQ_NON_CODING))
"inframe_altering",
NULL,
NULL,
- "coding_sequence"
+ "coding_sequence",
+ "feature_elongation",
+ "start_retained"
};
typedef struct
{
char *name; // human readable name, e.g. ORF45
- uint8_t iseq;
+ uint32_t iseq;
}
gf_gene_t;
typedef struct
{
bcf1_t *line;
uint32_t *smpl; // bitmask of sample consequences with first/second haplotype interleaved
- uint32_t nfmt:4, nvcsq:28, mvcsq;
+ uint32_t nfmt:4, // the bitmask size (the number of integers per sample)
+ nvcsq:28, mvcsq;
vcsq_t *vcsq; // there can be multiple consequences for a single VCF record
}
vrec_t;
{
vrec_t **vrec; // buffer of VCF lines with the same position
int n, m;
+ uint32_t keep_until; // the maximum transcript end position
};
KHASH_MAP_INIT_INT(pos2vbuf, vbuf_t*)
char *outdir, **argv, *fa_fname, *gff_fname, *output_fname;
char *bcsq_tag;
int argc, output_type;
- int phase, quiet, local_csq;
+ int phase, verbosity, local_csq, record_cmd_line;
int ncsq_max, nfmt_bcsq; // maximum number of csq per site that can be accessed from FORMAT/BCSQ
int ncsq_small_warned;
+ int brief_predictions;
int rid; // current chromosome
tr_heap_t *active_tr; // heap of active transcripts for quick flushing
int ncsq_buf, mcsq_buf;
id_tbl_t tscript_ids; // mapping between transcript id (eg. Zm00001d027245_T001) and a numeric idx
int force; // force run under various conditions. Currently only to skip out-of-phase transcripts
+ int n_threads; // extra compression/decompression threads
faidx_t *fai;
kstring_t str, str2;
aux->seq[aux->nseq] = strdup(chr_beg);
iseq = khash_str2int_inc(aux->seq2int, aux->seq[aux->nseq]);
aux->nseq++;
- assert( aux->nseq < 256 ); // see gf_gene_t.iseq
+ assert( aux->nseq < 1<<29 ); // see gf_gene_t.iseq and ftr_t.iseq
}
chr_end[1] = c;
return iseq;
int biotype = gff_parse_biotype(ss);
if ( biotype <= 0 )
{
- if ( !gff_ignored_biotype(args, ss) && args->quiet<2 ) fprintf(bcftools_stderr,"ignored transcript: %s\n",line);
+ if ( !gff_ignored_biotype(args, ss) && args->verbosity > 0 ) fprintf(bcftools_stderr,"ignored transcript: %s\n",line);
return;
}
int biotype = gff_parse_biotype(ss);
if ( biotype <= 0 )
{
- if ( !gff_ignored_biotype(args, ss) && args->quiet<2 ) fprintf(bcftools_stderr,"ignored gene: %s\n",line);
+ if ( !gff_ignored_biotype(args, ss) && args->verbosity > 0 ) fprintf(bcftools_stderr,"ignored gene: %s\n",line);
return;
}
if ( !ss ) return -1; // no ID, ignore the line
if ( !strncmp("chromosome",ss+3,10) ) return -1;
if ( !strncmp("supercontig",ss+3,11) ) return -1;
- if ( args->quiet<2 ) fprintf(bcftools_stderr,"ignored: %s\n", line);
+ if ( args->verbosity > 0 ) fprintf(bcftools_stderr,"ignored: %s\n", line);
return -1;
}
// 7. column: strand
if ( *ss == '+' ) ftr->strand = STRAND_FWD;
else if ( *ss == '-' ) ftr->strand = STRAND_REV;
- else { if ( args->quiet<2 ) fprintf(bcftools_stderr,"Skipping unknown strand: %c\n", *ss); return -1; }
+ else { if ( args->verbosity > 0 ) fprintf(bcftools_stderr,"Skipping unknown strand: %c\n", *ss); return -1; }
ss += 2;
// 8. column: phase (codon offset)
else if ( *ss == '1' ) ftr->phase = 1;
else if ( *ss == '2' ) ftr->phase = 2;
else if ( *ss == '.' ) ftr->phase = 0; // exons do not have phase
- else { if ( args->quiet<2 ) fprintf(bcftools_stderr,"Skipping unknown phase: %c, %s\n", *ss, line); return -1; }
+ else { if ( args->verbosity > 0 ) fprintf(bcftools_stderr,"Skipping unknown phase: %c, %s\n", *ss, line); return -1; }
ss += 2;
// substring search for "Parent=transcript:ENST00000437963"
{
if ( args->force )
{
- if ( args->quiet < 2 )
+ if ( args->verbosity > 0 )
fprintf(bcftools_stderr,"Warning: GFF3 assumption failed for transcript %s, CDS=%d: phase!=len%%3 (phase=%d, len=%d)\n",args->tscript_ids.str[tr->id],tr->cds[i]->beg+1,phase,len);
tscript_ok = 0;
break;
{
if ( args->force )
{
- if ( args->quiet < 2 )
+ if ( args->verbosity > 0 )
fprintf(bcftools_stderr,"Warning: GFF3 assumption failed for transcript %s, CDS=%d: phase!=len%%3 (phase=%d, len=%d)\n",args->tscript_ids.str[tr->id],tr->cds[i]->beg+1,phase,len);
tscript_ok = 0;
break;
}
tscript_init_cds(args);
- if ( !args->quiet )
+ if ( args->verbosity > 0 )
{
fprintf(bcftools_stderr,"Indexed %d transcripts, %d exons, %d CDSs, %d UTRs\n",
regidx_nregs(args->idx_tscript),
free(aux->seq);
gff_id_destroy(&aux->gene_ids);
- if ( args->quiet<2 && khash_str2int_size(aux->ignored_biotypes) )
+ if ( args->verbosity > 0 && khash_str2int_size(aux->ignored_biotypes) )
{
khash_t(str2int) *ign = (khash_t(str2int)*)aux->ignored_biotypes;
fprintf(bcftools_stderr,"Ignored the following biotypes:\n");
for (i = kh_begin(ign); i < kh_end(ign); i++)
{
if ( !kh_exist(ign,i)) continue;
- fprintf(bcftools_stderr,"\t%dx\t.. %s\n", kh_value(ign,i), kh_key(ign,i));
+ const char *biotype = kh_key(ign,i);
+ if ( !strcmp(biotype,"TCE") ) biotype = "TCE (\"To be Experimentally Confirmed\")";
+ fprintf(bcftools_stderr,"\t%dx\t.. %s\n", kh_value(ign,i), biotype);
}
}
khash_str2int_destroy_free(aux->ignored_biotypes);
{
args->nfmt_bcsq = 1 + (args->ncsq_max - 1) / 32;
- if ( !args->quiet ) fprintf(bcftools_stderr,"Parsing %s ...\n", args->gff_fname);
+ if ( args->verbosity > 0 ) fprintf(bcftools_stderr,"Parsing %s ...\n", args->gff_fname);
init_gff(args);
args->rid = -1;
if ( args->output_type==FT_TAB_TEXT )
{
// significant speedup for plain VCFs
- bcf_hdr_set_samples(args->hdr,NULL,0);
+ if (bcf_hdr_set_samples(args->hdr,NULL,0) < 0)
+ error_errno("[%s] Couldn't build sample filter", __func__);
}
args->phase = PHASE_DROP_GT;
}
if ( args->output_type==FT_TAB_TEXT )
{
args->out = args->output_fname ? fopen(args->output_fname,"w") : bcftools_stdout;
- if ( !args->out ) error("Failed to open %s: %s\n", args->output_fname,strerror(errno));
+ if ( !args->out ) error("Failed to write to %s: %s\n", !strcmp("-",args->output_fname)?"standard output":args->output_fname,strerror(errno));
fprintf(args->out,"# This file was produced by: bcftools +csq(%s+htslib-%s)\n", bcftools_version(),hts_version());
fprintf(args->out,"# The command line was:\tbcftools +%s", args->argv[0]);
else
{
args->out_fh = hts_open(args->output_fname? args->output_fname : "-",hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to %s: %s\n", args->output_fname? args->output_fname : "standard output", strerror(errno));
- bcf_hdr_append_version(args->hdr,args->argc,args->argv,"bcftools/csq");
- bcf_hdr_printf(args->hdr,"##INFO=<ID=%s,Number=.,Type=String,Description=\"%s consequence annotation from BCFtools/csq. Format: '[*]consequence|gene|transcript|biotype[|strand|amino_acid_change|dna_change]' or, for consequences of variants split across multiple sites, a pointer to the record storing the consequences '@position'. '*' prefix indicates a consequence downstream from a stop \">",args->bcsq_tag, args->local_csq ? "Local" : "Haplotype-aware");
+ if ( args->out_fh == NULL ) error("[%s] Error: cannot write to %s: %s\n", __func__,args->output_fname? args->output_fname : "standard output", strerror(errno));
+ if ( args->n_threads > 0)
+ hts_set_opt(args->out_fh, HTS_OPT_THREAD_POOL, args->sr->p);
+ if ( args->record_cmd_line ) bcf_hdr_append_version(args->hdr,args->argc,args->argv,"bcftools/csq");
+ bcf_hdr_printf(args->hdr,"##INFO=<ID=%s,Number=.,Type=String,Description=\"%s consequence annotation from BCFtools/csq, see http://samtools.github.io/bcftools/howtos/csq-calling.html for details. Format: Consequence|gene|transcript|biotype|strand|amino_acid_change|dna_change\">",args->bcsq_tag, args->local_csq ? "Local" : "Haplotype-aware");
if ( args->hdr_nsmpl )
bcf_hdr_printf(args->hdr,"##FORMAT=<ID=%s,Number=.,Type=Integer,Description=\"Bitmask of indexes to INFO/BCSQ, with interleaved first/second haplotype. Use \\\"bcftools query -f'[%%CHROM\\t%%POS\\t%%SAMPLE\\t%%TBCSQ\\n]'\\\" to translate.\">",args->bcsq_tag);
- bcf_hdr_write(args->out_fh, args->hdr);
+ if ( bcf_hdr_write(args->out_fh, args->hdr)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname?args->output_fname:"standard output");
}
- if ( !args->quiet ) fprintf(bcftools_stderr,"Calling...\n");
+ if ( args->verbosity > 0 ) fprintf(bcftools_stderr,"Calling...\n");
}
void destroy_data(args_t *args)
splice->vcf.pos = rec->pos;
splice->vcf.rlen = rec->rlen;
splice->vcf.ref = rec->d.allele[0];
+ splice->csq = 0;
}
static inline void splice_build_hap(splice_t *splice, uint32_t beg, int len)
{
#endif
}
void csq_stage(args_t *args, csq_t *csq, bcf1_t *rec);
-static inline int csq_stage_utr(args_t *args, regitr_t *itr, bcf1_t *rec, uint32_t trid)
+static inline int csq_stage_utr(args_t *args, regitr_t *itr, bcf1_t *rec, uint32_t trid, uint32_t type)
{
while ( regitr_overlap(itr) )
{
csq_t csq;
memset(&csq, 0, sizeof(csq_t));
csq.pos = rec->pos;
- csq.type.type = utr->which==prime5 ? CSQ_UTR5 : CSQ_UTR3;
+ csq.type.type = (utr->which==prime5 ? CSQ_UTR5 : CSQ_UTR3) | type;
csq.type.biotype = tr->type;
csq.type.strand = tr->strand;
csq.type.trid = tr->id;
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,splice->ref_beg+1,splice->ref_beg+1, itr) ) // adjacent utr
{
- ret = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ ret = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
if ( ret!=0 )
{
regitr_destroy(itr);
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,splice->ref_end-1,splice->ref_end-1, itr) ) // adjacent utr
{
- ret = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ ret = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
if ( ret!=0 )
{
regitr_destroy(itr);
return SPLICE_INSIDE;
}
+int shifted_del_synonymous(args_t *args, splice_t *splice, uint32_t ex_beg, uint32_t ex_end)
+{
+ static int small_ref_padding_warned = 0;
+ tscript_t *tr = splice->tr;
+
+ // We know the VCF record overlaps the exon, but does it overlap the start codon?
+ if ( tr->strand==STRAND_REV && splice->vcf.pos + splice->vcf.rlen + 2 <= ex_end ) return 0;
+ if ( tr->strand==STRAND_FWD && splice->vcf.pos >= ex_beg + 3 ) return 0;
+
+#if XDBG
+ fprintf(bcftools_stderr,"shifted_del_synonymous: %d-%d %s\n",ex_beg,ex_end, tr->strand==STRAND_FWD?"fwd":"rev");
+ fprintf(bcftools_stderr," %d .. %s > %s\n",splice->vcf.pos+1,splice->vcf.ref,splice->vcf.alt);
+#endif
+
+ // is there enough ref sequence for the extension? All coordinates are 0-based
+ int ref_len = strlen(splice->vcf.ref);
+ int alt_len = strlen(splice->vcf.alt);
+ assert( ref_len > alt_len );
+ int ndel = ref_len - alt_len;
+
+ if ( tr->strand==STRAND_REV )
+ {
+ int32_t vcf_ref_end = splice->vcf.pos + ref_len - 1; // end pos of the VCF REF allele
+ int32_t tr_ref_end = splice->tr->end + N_REF_PAD; // the end pos of accessible cached ref seq
+ if ( vcf_ref_end + ndel > tr_ref_end )
+ {
+ if ( !small_ref_padding_warned )
+ {
+ fprintf(bcftools_stderr,"Warning: Could not verify synonymous start/stop at %s:%d due to small N_REF_PAD. (Improve me?)\n",bcf_seqname(args->hdr,splice->vcf.rec),splice->vcf.pos+1);
+ small_ref_padding_warned = 1;
+ }
+ return 0;
+ }
+
+ char *ptr_vcf = splice->vcf.ref + alt_len; // the first deleted base in the VCF REF allele
+ char *ptr_ref = splice->tr->ref + N_REF_PAD + (vcf_ref_end + 1 - splice->tr->beg); // the first ref base after the ndel bases deleted
+#if XDBG
+ fprintf(bcftools_stderr,"vcf: %s\nref: %s\n",ptr_vcf,ptr_ref);
+#endif
+ int i = 0;
+ while ( ptr_vcf[i] && ptr_vcf[i]==ptr_ref[i] ) i++;
+ if ( ptr_vcf[i] ) return 0; // the deleted sequence cannot be replaced
+ }
+ else
+ {
+ // STRAND_FWD
+ int32_t vcf_block_beg = splice->vcf.pos + ref_len - 2*ndel; // the position of the first base of the ref block that could potentially replace the deletion
+ if ( vcf_block_beg < 0 ) return 0;
+
+#if XDBG
+ fprintf(bcftools_stderr,"vcf_block_beg: %d\n",vcf_block_beg+1);
+#endif
+
+ if ( N_REF_PAD + vcf_block_beg < ex_beg )
+ {
+ if ( !small_ref_padding_warned )
+ {
+ fprintf(bcftools_stderr,"Warning: Could not verify synonymous start/stop at %s:%d due to small N_REF_PAD. (Improve me?)\n",bcf_seqname(args->hdr,splice->vcf.rec),splice->vcf.pos+1);
+ small_ref_padding_warned = 1;
+ }
+ return 0;
+ }
+
+ char *ptr_vcf = splice->vcf.ref + alt_len; // the first deleted base in the VCF REF allele
+ char *ptr_ref = splice->tr->ref + N_REF_PAD + vcf_block_beg - splice->tr->beg; // the replacement ref block
+#if XDBG
+ fprintf(bcftools_stderr,"vcf: %s\nref: %s\n",ptr_vcf,ptr_ref);
+#endif
+
+ int i = 0;
+ while ( ptr_vcf[i] && ptr_vcf[i]==ptr_ref[i] ) i++;
+ if ( ptr_vcf[i] ) return 0; // the deleted sequence cannot be replaced
+ }
+
+ return 1;
+}
+
static inline int splice_csq_del(args_t *args, splice_t *splice, uint32_t ex_beg, uint32_t ex_end)
{
+ if ( splice->check_start )
+ {
+ // check for synonymous start
+ // test/csq/ENST00000375992/incorrect-synon-del-not-start-lost.txt
+ // test/csq/ENST00000368801.2/start-lost.txt
+ // test/csq/ENST00000318249.2/synonymous-start-lost.txt
+ int is_synonymous = shifted_del_synonymous(args, splice, ex_beg, ex_end);
+ if ( is_synonymous )
+ {
+ splice->csq |= CSQ_START_RETAINED;
+ return SPLICE_OVERLAP;
+ }
+ }
+
// coordinates that matter for consequences, eg AC>ACG trimmed to C>CG
splice->ref_beg = splice->vcf.pos + splice->tbeg - 1; // 1b before the deleted base
splice->ref_end = splice->vcf.pos + splice->vcf.rlen - splice->tend - 1; // the last deleted base
#if XDBG
-fprintf(bcftools_stderr,"del: %s>%s .. ex=%d,%d beg,end=%d,%d tbeg,tend=%d,%d check_utr=%d start,stop,beg,end=%d,%d,%d,%d\n", splice->vcf.ref,splice->vcf.alt,ex_beg,ex_end,splice->ref_beg,splice->ref_end,splice->tbeg,splice->tend,splice->check_utr,splice->check_start,splice->check_stop,splice->check_region_beg,splice->check_region_end);
+fprintf(bcftools_stderr,"splice_csq_del: %s>%s .. ex=%d,%d beg,end=%d,%d tbeg,tend=%d,%d check_utr=%d start,stop,beg,end=%d,%d,%d,%d\n", splice->vcf.ref,splice->vcf.alt,ex_beg,ex_end,splice->ref_beg,splice->ref_end,splice->tbeg,splice->tend,splice->check_utr,splice->check_start,splice->check_stop,splice->check_region_beg,splice->check_region_end);
#endif
if ( splice->ref_beg + 1 < ex_beg ) // the part before the exon; ref_beg is off by -1
regitr_t *itr = regitr_init(NULL);
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,splice->ref_beg,ex_beg-1, itr) ) // adjacent utr
- csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
regitr_destroy(itr);
}
if ( !csq )
regitr_t *itr = regitr_init(NULL);
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,ex_end+1,splice->ref_end, itr) ) // adjacent utr
- csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
regitr_destroy(itr);
}
if ( !csq )
csq_stage_splice(args, splice->vcf.rec, splice->tr, splice->csq);
return SPLICE_OUTSIDE;
}
-
if ( splice->ref_beg < ex_beg + 2 ) // ref_beg is off by -1
{
if ( splice->check_region_beg ) splice->csq |= CSQ_SPLICE_REGION;
regitr_t *itr = regitr_init(NULL);
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,splice->ref_beg,ex_beg-1, itr) ) // adjacent utr
- csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
regitr_destroy(itr);
}
if ( !csq )
regitr_t *itr = regitr_init(NULL);
const char *chr = bcf_seqname(args->hdr,splice->vcf.rec);
if ( regidx_overlap(args->idx_utr,chr,ex_end+1,splice->ref_end, itr) ) // adjacent utr
- csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id);
+ csq = csq_stage_utr(args, itr, splice->vcf.rec, splice->tr->id, splice->csq);
regitr_destroy(itr);
}
if ( !csq )
}
static inline int splice_csq(args_t *args, splice_t *splice, uint32_t ex_beg, uint32_t ex_end)
{
- splice->csq = 0;
splice->vcf.alen = strlen(splice->vcf.alt);
int rlen1 = splice->vcf.rlen - 1, alen1 = splice->vcf.alen - 1, i = 0;
return 0;
}
+
// return value: 0 added, 1 overlapping variant, 2 silent discard (intronic,alt=ref)
int hap_init(args_t *args, hap_node_t *parent, hap_node_t *child, gf_cds_t *cds, bcf1_t *rec, int ial)
{
if ( child->icds!=tr->ncds-1 ) splice.check_region_end = 1;
#if XDBG
-fprintf(bcftools_stderr,"\n%d [%s][%s] check start:%d,stop:%d\n",splice.vcf.pos+1,splice.vcf.ref,splice.vcf.alt,splice.check_start,splice.check_stop);
+fprintf(bcftools_stderr,"\nhap_init: %d [%s][%s] check start:%d,stop:%d\n",splice.vcf.pos+1,splice.vcf.ref,splice.vcf.alt,splice.check_start,splice.check_stop);
#endif
int ret = splice_csq(args, &splice, cds->beg, cds->beg + cds->len - 1);
#if XDBG
#endif
if ( ret==SPLICE_VAR_REF ) return 2; // not a variant, eg REF=CA ALT=CA
- if ( ret==SPLICE_OUTSIDE || ret==SPLICE_OVERLAP ) // not a coding csq
+ if ( ret==SPLICE_OUTSIDE || ret==SPLICE_OVERLAP || splice.csq==CSQ_START_LOST ) // not a coding csq
{
free(splice.kref.s);
free(splice.kalt.s);
if ( len < 0 ) // overlapping variants
{
free(str.s);
+ free(splice.kref.s);
+ free(splice.kalt.s);
return 1;
}
kputsn_(tr->ref + N_REF_PAD + parent->rbeg + parent->rlen - tr->beg, len, &str);
if ( !child->csq ) child->csq |= CSQ_CODING_SEQUENCE; // hack, specifically for ENST00000390520/deletion-overlap.vcf
}
+
free(splice.kref.s);
free(splice.kalt.s);
return 0;
void cds_translate(kstring_t *_ref, kstring_t *_seq, uint32_t sbeg, uint32_t rbeg, uint32_t rend, int strand, kstring_t *tseq, int fill)
{
#if XDBG
-fprintf(bcftools_stderr,"translate: %d %d %d fill=%d seq.l=%d\n",sbeg,rbeg,rend,fill,(int)_seq->l);
+fprintf(bcftools_stderr,"\ntranslate: %d %d %d fill=%d seq.l=%d\n",sbeg,rbeg,rend,fill,(int)_seq->l);
#endif
char tmp[3], *codon, *end;
int i, len, npad;
#if DBG>1
fprintf(bcftools_stderr," npad: %d\n",npad);
#endif
-if ( !(npad>=0 && sbeg+seq.l+npad<=seq.m) ) fprintf(bcftools_stderr,"sbeg=%d seq.l=%d seq.m=%d\n",sbeg,(int)seq.l,(int)seq.m);
+ if ( !(npad>=0 && sbeg+seq.l+npad<=seq.m) ) fprintf(bcftools_stderr,"sbeg=%d seq.l=%d seq.m=%d npad=%d\n",sbeg,(int)seq.l,(int)seq.m,npad);
assert( npad>=0 && sbeg+seq.l+npad<=seq.m ); // todo: first codon on the rev strand
if ( npad==2 )
for (; i>=0 && end>seq.s; i--) tmp[i] = *(--end);
#if DBG>1
fprintf(bcftools_stderr,"\t i=%d\n", i);
- if(i==1)fprintf(bcftools_stderr,"[0] %c\n",tmp[2]);
- if(i==0)fprintf(bcftools_stderr,"[0] %c%c\n",tmp[1],tmp[2]);
+ if(i==1)fprintf(bcftools_stderr,"[0] %c\n",tmp[2]);
+ if(i==0)fprintf(bcftools_stderr,"[0] %c%c\n",tmp[1],tmp[2]);
#endif
if ( i==-1 )
{
kputs(csq->vstr.s, str);
}
+void kprint_aa_prediction(args_t *args, int beg, kstring_t *aa, kstring_t *str)
+{
+ if ( !args->brief_predictions )
+ kputs(aa->s, str);
+ else
+ {
+ int len = aa->l;
+ if ( aa->s[len-1]=='*' ) len--;
+ kputc(aa->s[0], str);
+ kputs("..", str);
+ kputw(beg+len, str);
+ }
+}
+
void hap_add_csq(args_t *args, hap_t *hap, hap_node_t *node, int tlen, int ibeg, int iend, int dlen, int indel)
{
int i;
tscript_t *tr = hap->tr;
int ref_node = tr->strand==STRAND_FWD ? ibeg : iend;
-
int icsq = node->ncsq_list++;
hts_expand0(csq_t,node->ncsq_list,node->mcsq_list,node->csq_list);
csq_t *csq = &node->csq_list[icsq];
int aa_sbeg = tr->strand==STRAND_FWD ? node2sbeg(ibeg)/3+1 : (tlen - node2send(iend))/3+1;
kputc_('|', &str);
kputw(aa_rbeg, &str);
- kputs(hap->tref.s, &str);
+ kprint_aa_prediction(args,aa_rbeg,&hap->tref,&str);
if ( !(csq->type.type & CSQ_SYNONYMOUS_VARIANT) )
{
kputc_('>', &str);
kputw(aa_sbeg, &str);
- kputs(hap->tseq.s, &str);
+ kprint_aa_prediction(args,aa_sbeg,&hap->tseq,&str);
}
kputc_('|', &str);
int icsq = 2*csq->idx + ihap;
if ( icsq >= args->ncsq_max ) // more than ncsq_max consequences, so can't fit it in FMT
{
- int print_warning = 1;
- if ( args->quiet )
+ if ( args->verbosity && (!args->ncsq_small_warned || args->verbosity > 1) )
{
- if ( args->quiet > 1 || args->ncsq_small_warned ) print_warning = 0;
+ fprintf(bcftools_stderr,
+ "Warning: Too many consequences for sample %s at %s:%"PRId64", keeping the first %d and skipping the rest.\n",
+ args->hdr->samples[ismpl],bcf_hdr_id2name(args->hdr,args->rid),(int64_t) vrec->line->pos+1,csq->idx);
+ if ( !args->ncsq_small_warned )
+ fprintf(bcftools_stderr," The limit can be increased by setting the --ncsq parameter. This warning is printed only once.\n");
args->ncsq_small_warned = 1;
}
- if ( print_warning )
- {
- fprintf(bcftools_stderr,"Warning: --ncsq %d is too small to annotate %s at %s:%d with %d-th csq\n",
- args->ncsq_max/2,args->hdr->samples[ismpl],bcf_hdr_id2name(args->hdr,args->rid),vrec->line->pos+1,csq->idx+1);
- if ( args->quiet ) fprintf(bcftools_stderr,"(This warning is printed only once)\n");
- }
break;
}
if ( vrec->nfmt < 1 + icsq/32 ) vrec->nfmt = 1 + icsq/32;
{
int i,j;
tr_heap_t *heap = args->active_tr;
-
while ( heap->ndat && heap->dat[0]->end<=pos )
{
tscript_t *tr = heap->dat[0];
khp_delete(trhp, heap);
-
args->hap->tr = tr;
if ( tr->root && tr->root->nchild ) // normal, non-localized calling
{
#define SWAP(type_t, a, b) { type_t t = a; a = b; b = t; }
-void vbuf_push(args_t *args, bcf1_t **rec_ptr)
+vbuf_t *vbuf_push(args_t *args, bcf1_t **rec_ptr)
{
int i;
i = rbuf_append(&args->vcf_rbuf);
if ( !args->vcf_buf[i] ) args->vcf_buf[i] = (vbuf_t*) calloc(1,sizeof(vbuf_t));
args->vcf_buf[i]->n = 0;
+ args->vcf_buf[i]->keep_until = 0;
}
vbuf_t *vbuf = args->vcf_buf[i];
vbuf->n++;
int ret;
khint_t k = kh_put(pos2vbuf, args->pos2vbuf, (int)rec->pos, &ret);
kh_val(args->pos2vbuf,k) = vbuf;
+
+ return vbuf;
}
-void vbuf_flush(args_t *args)
+void vbuf_flush(args_t *args, uint32_t pos)
{
- if ( args->active_tr->ndat ) return; // cannot output buffered VCF lines (args.vbuf) until all active transcripts are gone
-
int i,j;
- while ( (i=rbuf_shift(&args->vcf_rbuf))>=0 )
+ while ( args->vcf_rbuf.n )
{
- vbuf_t *vbuf = args->vcf_buf[i];
+ vbuf_t *vbuf;
+ if ( !args->local_csq && args->active_tr->ndat )
+ {
+ // check if the first active transcript starts beyond the first buffered VCF record,
+ // cannot output buffered VCF lines (args.vbuf) until the active transcripts are gone
+ vbuf = args->vcf_buf[ args->vcf_rbuf.f ];
+ if ( vbuf->keep_until > pos ) break;
+ assert( vbuf->n );
+ }
+
+ i = rbuf_shift(&args->vcf_rbuf);
+ assert( i>=0 );
+ vbuf = args->vcf_buf[i];
+ int pos = vbuf->n ? vbuf->vrec[0]->line->pos : -1;
for (i=0; i<vbuf->n; i++)
{
vrec_t *vrec = vbuf->vrec[i];
}
if ( !vrec->nvcsq )
{
- bcf_write(args->out_fh, args->hdr, vrec->line);
+ if ( bcf_write(args->out_fh, args->hdr, vrec->line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname?args->output_fname:"standard output");
+ int save_pos = vrec->line->pos;
+ bcf_empty(vrec->line);
+ vrec->line->pos = save_pos; // this is necessary for compound variants
continue;
}
if ( args->hdr_nsmpl )
{
if ( vrec->nfmt < args->nfmt_bcsq )
- for (j=1; j<args->hdr_nsmpl; j++) memcpy(vrec->smpl+j*vrec->nfmt, vrec->smpl+j*args->nfmt_bcsq, vrec->nfmt*sizeof(*vrec->smpl));
+ for (j=1; j<args->hdr_nsmpl; j++)
+ memmove(&vrec->smpl[j*vrec->nfmt], &vrec->smpl[j*args->nfmt_bcsq], vrec->nfmt*sizeof(*vrec->smpl));
bcf_update_format_int32(args->hdr, vrec->line, args->bcsq_tag, vrec->smpl, args->hdr_nsmpl*vrec->nfmt);
}
vrec->nvcsq = 0;
- bcf_write(args->out_fh, args->hdr, vrec->line);
+ if ( bcf_write(args->out_fh, args->hdr, vrec->line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname?args->output_fname:"standard output");
+ int save_pos = vrec->line->pos;
+ bcf_empty(vrec->line);
+ vrec->line->pos = save_pos;
}
- if ( vbuf->n )
+ if ( pos!=-1 )
{
- khint_t k = kh_get(pos2vbuf, args->pos2vbuf, vbuf->vrec[0]->line->pos);
+ khint_t k = kh_get(pos2vbuf, args->pos2vbuf, pos);
if ( k != kh_end(args->pos2vbuf) ) kh_del(pos2vbuf, args->pos2vbuf, k);
}
vbuf->n = 0;
}
+ if ( args->active_tr->ndat ) return;
for (i=0; i<args->nrm_tr; i++)
{
int pad_end = len - (tr->end - tr->beg + 1 + pad_beg);
if ( pad_beg + pad_end != 2*N_REF_PAD )
{
- char *ref = (char*) malloc(tr->end - tr->beg + 1 + 2*N_REF_PAD);
+ char *ref = (char*) malloc(tr->end - tr->beg + 1 + 2*N_REF_PAD + 1);
for (i=0; i < N_REF_PAD - pad_beg; i++) ref[i] = 'N';
memcpy(ref+i, tr->ref, len);
+ len += i;
for (i=0; i < N_REF_PAD - pad_end; i++) ref[i+len] = 'N';
+ ref[i+len] = 0;
free(tr->ref);
tr->ref = ref;
}
static void sanity_check_ref(args_t *args, tscript_t *tr, bcf1_t *rec)
{
- char *ref = tr->ref + (rec->pos + N_REF_PAD >= tr->beg ? rec->pos - tr->beg + N_REF_PAD : 0);
- char *vcf = rec->d.allele[0] + (rec->pos + N_REF_PAD >= tr->beg ? 0 : tr->beg - N_REF_PAD - rec->pos);
- assert( vcf - rec->d.allele[0] < strlen(rec->d.allele[0]) );
- while ( *ref && *vcf )
+ int vbeg = 0;
+ int rbeg = rec->pos - tr->beg + N_REF_PAD;
+ if ( rbeg < 0 ) { vbeg += abs(rbeg); rbeg = 0; }
+ char *ref = tr->ref + rbeg;
+ char *vcf = rec->d.allele[0] + vbeg;
+ assert( vcf - rec->d.allele[0] < strlen(rec->d.allele[0]) && ref - tr->ref < tr->end - tr->beg + 2*N_REF_PAD );
+ int i = 0;
+ while ( ref[i] && vcf[i] )
{
- if ( *ref!=*vcf && toupper(*ref)!=toupper(*vcf) )
- error("Error: the fasta reference does not match the VCF REF allele at %s:%d .. %s\n", bcf_seqname(args->hdr,rec),rec->pos+1,rec->d.allele[0]);
- ref++;
- vcf++;
+ if ( ref[i]!=vcf[i] && toupper(ref[i])!=toupper(vcf[i]) )
+ error("Error: the fasta reference does not match the VCF REF allele at %s:%"PRId64" .. fasta=%c vcf=%c\n",
+ bcf_seqname(args->hdr,rec),(int64_t) rec->pos+vbeg+1,ref[i],vcf[i]);
+ i++;
}
}
for (i=1; i<rec->n_allele; i++)
{
+ if ( rec->d.allele[i][0]=='<' || rec->d.allele[i][0]=='*' ) { continue; }
if ( hap_init(args, &root, &node, cds, rec, i)!=0 ) continue;
csq_t csq;
int aa_sbeg = tr->strand==STRAND_FWD ? node.sbeg/3+1 : (tr->nsref - 2*N_REF_PAD + node.dlen - node.sbeg - alen)/3+1;
kputc_('|', &str);
kputw(aa_rbeg, &str);
- kputs(tref->s, &str);
+ kprint_aa_prediction(args,aa_rbeg,tref,&str);
if ( !(csq_type & CSQ_SYNONYMOUS_VARIANT) )
{
kputc_('>', &str);
kputw(aa_sbeg, &str);
- kputs(tseq->s, &str);
+ kprint_aa_prediction(args,aa_sbeg,tseq,&str);
}
kputc_('|', &str);
kputw(rec->pos+1, &str);
return ret;
}
-int test_cds(args_t *args, bcf1_t *rec)
+int test_cds(args_t *args, bcf1_t *rec, vbuf_t *vbuf)
{
+ static int overlaps_warned = 0, multiploid_warned = 0;
+
int i, ret = 0, hap_ret;
const char *chr = bcf_seqname(args->hdr,rec);
// note that the off-by-one extension of rlen is deliberate to account for insertions
gf_cds_t *cds = regitr_payload(args->itr,gf_cds_t*);
tscript_t *tr = cds->tr;
if ( !GF_is_coding(tr->type) ) continue;
+ if ( vbuf->keep_until < tr->end ) vbuf->keep_until = tr->end;
ret = 1;
if ( !tr->root )
{
// overlapping or intron variant, cannot apply
if ( hap_ret==1 )
{
- if ( !args->quiet )
- fprintf(bcftools_stderr,"Warning: Skipping overlapping variants at %s:%d\t%s>%s\n", chr,rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ if ( args->verbosity && (!overlaps_warned || args->verbosity > 1) )
+ {
+ fprintf(bcftools_stderr,
+ "Warning: Skipping overlapping variants at %s:%"PRId64"\t%s>%s.\n",
+ chr,(int64_t) rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ if ( !overlaps_warned )
+ fprintf(bcftools_stderr," This message is printed only once, the verbosity can be increased with `--verbose 2`\n");
+ overlaps_warned = 1;
+ }
if ( args->out )
- fprintf(args->out,"LOG\tWarning: Skipping overlapping variants at %s:%d\t%s>%s\n", chr,rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ fprintf(args->out,"LOG\tWarning: Skipping overlapping variants at %s:%"PRId64"\t%s>%s\n", chr,(int64_t) rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
}
else ret = 1; // prevent reporting as intron in test_tscript
hap_destroy(child);
ngts /= bcf_hdr_nsamples(args->hdr);
if ( ngts!=1 && ngts!=2 )
{
- if ( !args->quiet )
- fprintf(bcftools_stderr,"Warning: Skipping site with non-diploid/non-haploid genotypes at %s:%d\t%s>%s\n", chr,rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ if ( args->verbosity && (!multiploid_warned || args->verbosity > 1) )
+ {
+ fprintf(bcftools_stderr,
+ "Warning: Skipping site with non-diploid/non-haploid genotypes at %s:%"PRId64"\t%s>%s.\n",
+ chr,(int64_t) rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ if ( !multiploid_warned )
+ fprintf(bcftools_stderr," This message is printed only once, the verbosity can be increased with `--verbose 2`\n");
+ multiploid_warned = 1;
+ }
if ( args->out )
- fprintf(args->out,"LOG\tWarning: Skipping site with non-diploid/non-haploid genotypes at %s:%d\t%s>%s\n", chr,rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
+ fprintf(args->out,"LOG\tWarning: Skipping site with non-diploid/non-haploid genotypes at %s:%"PRId64"\t%s>%s\n", chr,(int64_t) rec->pos+1,rec->d.allele[0],rec->d.allele[1]);
continue;
}
for (ismpl=0; ismpl<args->smpl->n; ismpl++)
if ( !bcf_gt_is_phased(gt[0]) && !bcf_gt_is_phased(gt[1]) )
{
if ( args->phase==PHASE_REQUIRE )
- error("Unphased heterozygous genotype at %s:%d, sample %s. See the --phase option.\n", chr,rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]]);
+ error("Unphased heterozygous genotype at %s:%"PRId64", sample %s. See the --phase option.\n", chr,(int64_t) rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]]);
if ( args->phase==PHASE_SKIP )
continue;
if ( args->phase==PHASE_NON_REF )
// overlapping or intron variant, cannot apply
if ( hap_ret==1 )
{
- if ( !args->quiet )
- fprintf(bcftools_stderr,"Warning: Skipping overlapping variants at %s:%d, sample %s\t%s>%s\n",
- chr,rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]],rec->d.allele[0],rec->d.allele[ial]);
+ if ( args->verbosity && (!overlaps_warned || args->verbosity > 1) )
+ {
+ fprintf(bcftools_stderr,
+ "Warning: Skipping overlapping variants at %s:%"PRId64", sample %s\t%s>%s.\n",
+ chr,(int64_t) rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]],rec->d.allele[0],rec->d.allele[ial]);
+ if ( !overlaps_warned )
+ fprintf(bcftools_stderr," This message is printed only once, the verbosity can be increased with `--verbose 2`\n");
+ overlaps_warned = 1;
+ }
if ( args->out )
- fprintf(args->out,"LOG\tWarning: Skipping overlapping variants at %s:%d, sample %s\t%s>%s\n",
- chr,rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]],rec->d.allele[0],rec->d.allele[ial]);
+ fprintf(args->out,"LOG\tWarning: Skipping overlapping variants at %s:%"PRId64", sample %s\t%s>%s\n",
+ chr,(int64_t) rec->pos+1,args->hdr->samples[args->smpl->idx[ismpl]],rec->d.allele[0],rec->d.allele[ial]);
}
hap_destroy(child);
continue;
if ( icsq >= args->ncsq_max ) // more than ncsq_max consequences, so can't fit it in FMT
{
int ismpl = args->smpl->idx[i];
- int print_warning = 1;
- if ( args->quiet )
+ if ( args->verbosity && (!args->ncsq_small_warned || args->verbosity > 1) )
{
- if ( args->quiet > 1 || args->ncsq_small_warned ) print_warning = 0;
+ fprintf(bcftools_stderr,
+ "Warning: Too many consequences for sample %s at %s:%"PRId64", keeping the first %d and skipping the rest.\n",
+ args->hdr->samples[ismpl],bcf_hdr_id2name(args->hdr,args->rid),(int64_t) vrec->line->pos+1,icsq+1);
+ if ( !args->ncsq_small_warned )
+ fprintf(bcftools_stderr," The limit can be increased by setting the --ncsq parameter. This warning is printed only once.\n");
args->ncsq_small_warned = 1;
}
- if ( print_warning )
- {
- fprintf(bcftools_stderr,"Warning: --ncsq %d is too small to annotate %s at %s:%d with %d-th csq\n",
- args->ncsq_max/2,args->hdr->samples[ismpl],bcf_hdr_id2name(args->hdr,args->rid),vrec->line->pos+1,csq->idx+1);
- if ( args->quiet ) fprintf(bcftools_stderr,"(This warning is printed only once)\n");
- }
- break;
}
if ( vrec->nfmt < 1 + icsq/32 ) vrec->nfmt = 1 + icsq/32;
vrec->smpl[i*args->nfmt_bcsq + icsq/32] |= 1 << (icsq % 32);
tscript_t *tr = splice.tr = utr->tr;
for (i=1; i<rec->n_allele; i++)
{
- if ( rec->d.allele[1][0]=='<' || rec->d.allele[1][0]=='*' ) { continue; }
+ if ( rec->d.allele[i][0]=='<' || rec->d.allele[i][0]=='*' ) { continue; }
splice.vcf.alt = rec->d.allele[i];
+ splice.csq = 0;
int splice_ret = splice_csq(args, &splice, utr->beg, utr->end);
if ( splice_ret!=SPLICE_INSIDE && splice_ret!=SPLICE_OVERLAP ) continue;
csq_t csq;
{
if ( rec->d.allele[1][0]=='<' || rec->d.allele[1][0]=='*' ) { continue; }
splice.vcf.alt = rec->d.allele[i];
+ splice.csq = 0;
splice_csq(args, &splice, exon->beg, exon->end);
if ( splice.csq ) ret = 1;
}
tscript_t *tr = splice.tr = regitr_payload(args->itr, tscript_t*);
for (i=1; i<rec->n_allele; i++)
{
- if ( rec->d.allele[1][0]=='<' || rec->d.allele[1][0]=='*' ) { continue; }
+ if ( rec->d.allele[i][0]=='<' || rec->d.allele[i][0]=='*' ) { continue; }
splice.vcf.alt = rec->d.allele[i];
+ splice.csq = 0;
int splice_ret = splice_csq(args, &splice, tr->beg, tr->end);
if ( splice_ret!=SPLICE_INSIDE && splice_ret!=SPLICE_OVERLAP ) continue; // SPLICE_OUTSIDE or SPLICE_REF
csq_t csq;
return ret;
}
-void process(args_t *args, bcf1_t **rec_ptr)
+void test_symbolic_alt(args_t *args, bcf1_t *rec)
+{
+ static int warned = 0;
+ if ( args->verbosity && (!warned && args->verbosity > 0) )
+ {
+ fprintf(bcftools_stderr,"Warning: The support for symbolic ALT insertions is experimental.\n");
+ warned = 1;
+ }
+
+ const char *chr = bcf_seqname(args->hdr,rec);
+
+ // only insertions atm
+ int beg = rec->pos + 1;
+ int end = beg;
+ int csq_class = CSQ_ELONGATION;
+
+ int hit = 0;
+ if ( regidx_overlap(args->idx_cds,chr,beg,end, args->itr) )
+ {
+ while ( regitr_overlap(args->itr) )
+ {
+ csq_t csq;
+ memset(&csq, 0, sizeof(csq_t));
+ gf_cds_t *cds = regitr_payload(args->itr,gf_cds_t*);
+ tscript_t *tr = cds->tr;
+ csq.type.type = (GF_is_coding(tr->type) ? CSQ_CODING_SEQUENCE : CSQ_NON_CODING) | csq_class;
+ csq.pos = rec->pos;
+ csq.type.biotype = tr->type;
+ csq.type.strand = tr->strand;
+ csq.type.trid = tr->id;
+ csq.type.gene = tr->gene->name;
+ csq_stage(args, &csq, rec);
+ hit = 1;
+ }
+ }
+ if ( regidx_overlap(args->idx_utr,chr,beg,end, args->itr) )
+ {
+ while ( regitr_overlap(args->itr) )
+ {
+ csq_t csq;
+ memset(&csq, 0, sizeof(csq_t));
+ gf_utr_t *utr = regitr_payload(args->itr, gf_utr_t*);
+ tscript_t *tr = utr->tr;
+ csq.type.type = (utr->which==prime5 ? CSQ_UTR5 : CSQ_UTR3) | csq_class;
+ csq.pos = rec->pos;
+ csq.type.biotype = tr->type;
+ csq.type.strand = tr->strand;
+ csq.type.trid = tr->id;
+ csq.type.gene = tr->gene->name;
+ csq_stage(args, &csq, rec);
+ hit = 1;
+ }
+ }
+ if ( regidx_overlap(args->idx_exon,chr,beg,end, args->itr) )
+ {
+ splice_t splice;
+ splice_init(&splice, rec);
+ splice.check_acceptor = splice.check_donor = 1;
+
+ while ( regitr_overlap(args->itr) )
+ {
+ gf_exon_t *exon = regitr_payload(args->itr, gf_exon_t*);
+ splice.tr = exon->tr;
+ if ( !splice.tr->ncds ) continue; // not a coding transcript, no interest in splice sites
+ splice.check_region_beg = splice.tr->beg==exon->beg ? 0 : 1;
+ splice.check_region_end = splice.tr->end==exon->end ? 0 : 1;
+ splice.vcf.alt = rec->d.allele[1];
+ splice.csq = csq_class;
+ splice_csq(args, &splice, exon->beg, exon->end);
+ if ( splice.csq ) hit = 1;
+ }
+ }
+ if ( !hit && regidx_overlap(args->idx_tscript,chr,beg,end, args->itr) )
+ {
+ splice_t splice;
+ splice_init(&splice, rec);
+
+ while ( regitr_overlap(args->itr) )
+ {
+ csq_t csq;
+ memset(&csq, 0, sizeof(csq_t));
+ tscript_t *tr = splice.tr = regitr_payload(args->itr, tscript_t*);
+ splice.vcf.alt = rec->d.allele[1];
+ splice.csq = csq_class;
+ int splice_ret = splice_csq(args, &splice, tr->beg, tr->end);
+ if ( splice_ret!=SPLICE_INSIDE && splice_ret!=SPLICE_OVERLAP ) continue; // SPLICE_OUTSIDE or SPLICE_REF
+ csq.type.type = (GF_is_coding(tr->type) ? CSQ_INTRON : CSQ_NON_CODING) | csq_class;
+ csq.pos = rec->pos;
+ csq.type.biotype = tr->type;
+ csq.type.strand = tr->strand;
+ csq.type.trid = tr->id;
+ csq.type.gene = tr->gene->name;
+ csq_stage(args, &csq, rec);
+ }
+ }
+}
+
+void debug_print_buffers(args_t *args, int pos)
+{
+ int i,j;
+ fprintf(bcftools_stderr,"debug_print_buffers at %d\n", pos);
+ fprintf(bcftools_stderr,"vbufs:\n");
+ for (i=0; i<args->vcf_rbuf.n; i++)
+ {
+ int k = rbuf_kth(&args->vcf_rbuf, i);
+ vbuf_t *vbuf = args->vcf_buf[k];
+
+ fprintf(bcftools_stderr,"\tvbuf %d:\n", i);
+ for (j=0; j<vbuf->n; j++)
+ {
+ vrec_t *vrec = vbuf->vrec[j];
+ fprintf(bcftools_stderr,"\t\t%"PRId64" .. nvcsq=%d\n", (int64_t) vrec->line->pos+1, vrec->nvcsq);
+ }
+ }
+ fprintf(bcftools_stderr,"pos2vbuf:");
+ khint_t k;
+ for (k = 0; k < kh_end(args->pos2vbuf); ++k)
+ if (kh_exist(args->pos2vbuf, k)) fprintf(bcftools_stderr," %d",1+(int)kh_key(args->pos2vbuf, k));
+ fprintf(bcftools_stderr,"\n");
+ fprintf(bcftools_stderr,"active_tr: %d\n", args->active_tr->ndat);
+}
+
+static void process(args_t *args, bcf1_t **rec_ptr)
{
if ( !rec_ptr )
{
hap_flush(args, REGIDX_MAX);
- vbuf_flush(args);
+ vbuf_flush(args, REGIDX_MAX);
return;
}
bcf1_t *rec = *rec_ptr;
+ static int32_t prev_rid = -1, prev_pos = -1;
+ if ( prev_rid!=rec->rid ) { prev_rid = rec->rid; prev_pos = rec->pos; }
+ if ( prev_pos > rec->pos )
+ error("Error: The file is not sorted, %s:%d comes before %s:%"PRId64"\n",bcf_seqname(args->hdr,rec),prev_pos+1,bcf_seqname(args->hdr,rec),(int64_t) rec->pos+1);
int call_csq = 1;
- if ( !rec->n_allele ) call_csq = 0; // no alternate allele
- else if ( rec->n_allele==2 && (rec->d.allele[1][0]=='<' || rec->d.allele[1][0]=='*') ) call_csq = 0; // gVCF, no alt allele
- else if ( rec->d.allele[1][0]=='<' && rec->d.allele[1][0]!='*') call_csq = 0; // a symbolic allele, not ready for CNVs etc
- else if ( args->filter )
+ if ( rec->n_allele < 2 ) call_csq = 0; // no alternate allele
+ else if ( rec->n_allele==2 && (rec->d.allele[1][0]=='*' || rec->d.allele[1][1]=='*') ) call_csq = 0; // gVCF, not an alt allele
+ else if ( rec->d.allele[1][0]=='<' )
+ {
+ if ( strncmp("<INS",rec->d.allele[1], 4) ) call_csq = 0; // only <INS[:.*]> is supported at the moment
+ }
+ if ( call_csq && args->filter )
{
call_csq = filter_test(args->filter, rec, NULL);
if ( args->filter_logic==FLT_EXCLUDE ) call_csq = call_csq ? 0 : 1;
{
if ( !args->out_fh ) return; // not a VCF output
vbuf_push(args, rec_ptr);
- vbuf_flush(args);
+ hap_flush(args, rec->pos-1);
+ vbuf_flush(args, rec->pos-1);
return;
}
if ( args->rid != rec->rid )
{
hap_flush(args, REGIDX_MAX);
- vbuf_flush(args);
+ vbuf_flush(args, REGIDX_MAX);
}
args->rid = rec->rid;
- vbuf_push(args, rec_ptr);
+ vbuf_t *vbuf = vbuf_push(args, rec_ptr);
- int hit = args->local_csq ? test_cds_local(args, rec) : test_cds(args, rec);
- hit += test_utr(args, rec);
- hit += test_splice(args, rec);
- if ( !hit ) test_tscript(args, rec);
+ if ( rec->d.allele[1][0]!='<' )
+ {
+ int hit = args->local_csq ? test_cds_local(args, rec) : test_cds(args, rec, vbuf);
+ hit += test_utr(args, rec);
+ hit += test_splice(args, rec);
+ if ( !hit ) test_tscript(args, rec);
+ }
+ else
+ test_symbolic_alt(args, rec);
- hap_flush(args, rec->pos-1);
- vbuf_flush(args);
+ if ( rec->pos > 0 )
+ {
+ hap_flush(args, rec->pos-1);
+ vbuf_flush(args, rec->pos-1);
+ }
return;
}
" -g, --gff-annot <file> gff3 annotation file\n"
"\n"
"CSQ options:\n"
+ " -b, --brief-predictions annotate with abbreviated protein-changing predictions\n"
" -c, --custom-tag <string> use this tag instead of the default BCSQ\n"
" -l, --local-csq localized predictions, consider only one VCF record at a time\n"
" -n, --ncsq <int> maximum number of consequences to consider per site [16]\n"
" -e, --exclude <expr> exclude sites for which the expression is true\n"
" --force run even if some sanity checks fail\n"
" -i, --include <expr> select sites for which the expression is true\n"
+ " --no-version do not append version and command line to the header\n"
" -o, --output <file> write output to a file [standard output]\n"
" -O, --output-type <b|u|z|v|t> b: compressed BCF, u: uncompressed BCF, z: compressed VCF\n"
" v: uncompressed VCF, t: plain tab-delimited text output [v]\n"
- " -q, --quiet suppress warning messages. Can be given two times for even less messages\n"
" -r, --regions <region> restrict to comma-separated list of regions\n"
" -R, --regions-file <file> restrict to regions listed in a file\n"
" -s, --samples <-|list> samples to include or \"-\" to apply all variants and ignore samples\n"
" -S, --samples-file <file> samples to include\n"
" -t, --targets <region> similar to -r but streams rather than index-jumps\n"
" -T, --targets-file <file> similar to -R but streams rather than index-jumps\n"
+ " --threads <int> use multithreading with <int> worker threads [0]\n"
+ " -v, --verbose <int> verbosity level 0-2 [1]\n"
"\n"
"Example:\n"
" bcftools csq -f hs37d5.fa -g Homo_sapiens.GRCh37.82.gff3.gz in.vcf\n"
args->output_type = FT_VCF;
args->bcsq_tag = "BCSQ";
args->ncsq_max = 2*16;
+ args->verbosity = 1;
+ args->record_cmd_line = 1;
static struct option loptions[] =
{
{"force",0,0,1},
+ {"threads",required_argument,NULL,2},
{"help",0,0,'h'},
{"ncsq",1,0,'n'},
+ {"brief-predictions",0,0,'b'},
{"custom-tag",1,0,'c'},
{"local-csq",0,0,'l'},
{"gff-annot",1,0,'g'},
{"output-type",1,NULL,'O'},
{"phase",1,0,'p'},
{"quiet",0,0,'q'},
+ {"verbose",1,0,'v'},
{"regions",1,0,'r'},
{"regions-file",1,0,'R'},
{"samples",1,0,'s'},
{"samples-file",1,0,'S'},
{"targets",1,0,'t'},
{"targets-file",1,0,'T'},
+ {"no-version",no_argument,NULL,3},
{0,0,0,0}
};
int c, targets_is_file = 0, regions_is_file = 0;
- char *targets_list = NULL, *regions_list = NULL;
- while ((c = getopt_long(argc, argv, "?hr:R:t:T:i:e:f:o:O:g:s:S:p:qc:ln:",loptions,NULL)) >= 0)
+ char *targets_list = NULL, *regions_list = NULL, *tmp;
+ while ((c = getopt_long(argc, argv, "?hr:R:t:T:i:e:f:o:O:g:s:S:p:qc:ln:bv:",loptions,NULL)) >= 0)
{
switch (c)
{
case 1 : args->force = 1; break;
+ case 2 :
+ args->n_threads = strtol(optarg,&tmp,10);
+ if ( *tmp ) error("Could not parse argument: --threads %s\n", optarg);
+ break;
+ case 3 : args->record_cmd_line = 0; break;
+ case 'b': args->brief_predictions = 1; break;
case 'l': args->local_csq = 1; break;
case 'c': args->bcsq_tag = optarg; break;
- case 'q': args->quiet++; break;
+ case 'q': error("Error: the -q option has been deprecated, use -v, --verbose instead.\n"); break;
+ case 'v':
+ args->verbosity = atoi(optarg);
+ if ( args->verbosity<0 || args->verbosity>2 ) error("Error: expected integer 0-2 with -v, --verbose\n");
+ break;
case 'p':
switch (optarg[0])
{
error("Failed to read the targets: %s\n", targets_list);
if ( regions_list && bcf_sr_set_regions(args->sr, regions_list, regions_is_file)<0 )
error("Failed to read the regions: %s\n", regions_list);
+ if ( bcf_sr_set_threads(args->sr, args->n_threads)<0 ) error("Failed to create %d extra threads\n", args->n_threads);
if ( !bcf_sr_add_reader(args->sr, fname) )
- error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->sr->errnum));
+ error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->sr->errnum));
args->hdr = bcf_sr_get_header(args->sr,0);
init_data(args);
destroy_data(args);
bcf_sr_destroy(args->sr);
free(args);
-
return 0;
}
#include <errno.h>
#include <math.h>
#include <sys/types.h>
+#include <inttypes.h>
+#ifndef _WIN32
#include <pwd.h>
+#endif
#include <regex.h>
#include <htslib/khash_str2int.h>
#include <htslib/hts_defs.h>
# define __FUNCTION__ __func__
#endif
-uint64_t bcf_double_missing = 0x7ff0000000000001;
-uint64_t bcf_double_vector_end = 0x7ff0000000000002;
+static const uint64_t bcf_double_missing = 0x7ff0000000000001;
+static const uint64_t bcf_double_vector_end = 0x7ff0000000000002;
static inline void bcf_double_set(double *ptr, uint64_t value)
{
union { uint64_t i; double d; } u;
#define bcf_double_set_missing(x) bcf_double_set(&(x),bcf_double_missing)
#define bcf_double_is_vector_end(x) bcf_double_test((x),bcf_double_vector_end)
#define bcf_double_is_missing(x) bcf_double_test((x),bcf_double_missing)
+#define bcf_double_is_missing_or_vector_end(x) (bcf_double_test((x),bcf_double_missing) || bcf_double_test((x),bcf_double_vector_end))
typedef struct _token_t
char *tag; // for debugging and printout only, VCF tag name
double threshold; // filtering threshold
int is_constant; // the threshold is set
- int hdr_id, type; // BCF header lookup ID and one of BCF_HT_* types
+ int hdr_id, tag_type; // BCF header lookup ID and one of BCF_HL_* types
int idx; // 0-based index to VCF vectors,
// -2: list (e.g. [0,1,2] or [1..3] or [1..] or any field[*], which is equivalent to [0..])
int *idxs; // set indexes to 0 to exclude, to 1 to include, and last element negative if unlimited
#define TOK_CNT 26
#define TOK_PERLSUB 27
#define TOK_BINOM 28
+#define TOK_PHRED 29
+#define TOK_MEDIAN 30
+#define TOK_STDEV 31
-// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
-// ( ) [ < = > ] ! | & + - * / M m a A O ~ ^ S . l f c p
-static int op_prec[] = {0,1,1,5,5,5,5,5,5,2,3, 6, 6, 7, 7, 8, 8, 8, 3, 2, 5, 5, 8, 8, 8, 8, 8, 8};
-#define TOKEN_STRING "x()[<=>]!|&+-*/MmaAO~^S.lfcp"
+// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
+// ( ) [ < = > ] ! | & + - * / M m a A O ~ ^ S . l f c p b P i s
+static int op_prec[] = {0,1,1,5,5,5,5,5,5,2,3, 6, 6, 7, 7, 8, 8, 8, 3, 2, 5, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};
+#define TOKEN_STRING "x()[<=>]!|&+-*/MmaAO~^S.lfcpis"
// Return negative values if it is a function with variable number of arguments
static int filters_next_token(char **str, int *len)
if ( !strncasecmp(tmp,"MAX(",4) ) { (*str) += 3; return TOK_MAX; }
if ( !strncasecmp(tmp,"MIN(",4) ) { (*str) += 3; return TOK_MIN; }
+ if ( !strncasecmp(tmp,"MEAN(",5) ) { (*str) += 4; return TOK_AVG; }
+ if ( !strncasecmp(tmp,"MEDIAN(",7) ) { (*str) += 6; return TOK_MEDIAN; }
if ( !strncasecmp(tmp,"AVG(",4) ) { (*str) += 3; return TOK_AVG; }
+ if ( !strncasecmp(tmp,"STDEV(",6) ) { (*str) += 5; return TOK_STDEV; }
if ( !strncasecmp(tmp,"SUM(",4) ) { (*str) += 3; return TOK_SUM; }
if ( !strncasecmp(tmp,"ABS(",4) ) { (*str) += 3; return TOK_ABS; }
if ( !strncasecmp(tmp,"COUNT(",4) ) { (*str) += 5; return TOK_CNT; }
if ( !strncasecmp(tmp,"STRLEN(",7) ) { (*str) += 6; return TOK_LEN; }
if ( !strncasecmp(tmp,"BINOM(",6) ) { (*str) += 5; return -TOK_BINOM; }
+ if ( !strncasecmp(tmp,"PHRED(",6) ) { (*str) += 5; return TOK_PHRED; }
if ( !strncasecmp(tmp,"%MAX(",5) ) { (*str) += 4; return TOK_MAX; } // for backward compatibility
if ( !strncasecmp(tmp,"%MIN(",5) ) { (*str) += 4; return TOK_MIN; } // for backward compatibility
if ( !strncasecmp(tmp,"%AVG(",5) ) { (*str) += 4; return TOK_AVG; } // for backward compatibility
if ( !strncasecmp(tmp,"PERL.",5) ) { (*str) += 5; return -TOK_PERLSUB; }
if ( !strncasecmp(tmp,"N_PASS(",7) ) { *len = 6; (*str) += 6; return -TOK_FUNC; }
if ( !strncasecmp(tmp,"F_PASS(",7) ) { *len = 6; (*str) += 6; return -TOK_FUNC; }
+ if ( !strncasecmp(tmp,"%ILEN",5) ) { *len = 5; return TOK_VAL; } // to be able to distinguish between INFO/ILEN and on-the-fly ILEN
if ( tmp[0]=='@' ) // file name
{
}
-/*
+/*
Simple path expansion, expands ~/, ~user, $var. The result must be freed by the caller.
- Based on jkb's staden code with some adjustements.
+ Based on jkb's staden code with some adjustments.
https://sourceforge.net/p/staden/code/HEAD/tree/staden/trunk/src/Misc/getfile.c#l123
*/
char *expand_path(char *path)
{
-#ifdef _WIN32
- return strdup(path); // windows expansion: todo
-#endif
-
kstring_t str = {0,0,0};
if ( path[0] == '~' )
{
if ( !path[1] || path[1] == '/' )
{
+#ifdef _WIN32
+ kputs(getenv("HOMEDRIVE"), &str);
+ kputs(getenv("HOMEPATH"), &str);
+#else
// ~ or ~/path
kputs(getenv("HOME"), &str);
if ( path[1] ) kputs(path+1, &str);
+#endif
}
+#ifndef _WIN32
else
{
// user name: ~pd3/path
else kputs(pwentry->pw_dir, &str);
kputs(end, &str);
}
- return str.s;
+#endif
+ return ks_release(&str);
}
if ( path[0] == '$' )
{
char *var = getenv(path+1);
- if ( var ) path = var;
+ if ( var ) {
+ kputs(var, &str);
+ return ks_release(&str);
+ }
}
+
return strdup(path);
}
return;
}
+ if ( !btok->str_value.l ) error("Error occurred while evaluating the expression\n");
+
if ( rtok->tok_type==TOK_EQ )
rtok->pass_site = strcmp(btok->str_value.s,line->d.id) ? 0 : 1;
else
return -1; // this shouldn't happen
}
+static void filters_set_chrom(filter_t *flt, bcf1_t *line, token_t *tok)
+{
+ tok->str_value.l = 0;
+ kputs(bcf_seqname(flt->hdr,line), &tok->str_value);
+ tok->nvalues = tok->str_value.l;
+ tok->is_str = 1;
+}
+
static void filters_set_pos(filter_t *flt, bcf1_t *line, token_t *tok)
{
tok->values[0] = line->pos+1;
static void filters_set_format_int(filter_t *flt, bcf1_t *line, token_t *tok)
{
if ( line->n_sample != tok->nsamples )
- error("Incorrect number of FORMAT fields at %s:%d .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),line->pos+1,tok->tag,line->n_sample,tok->nsamples);
+ error("Incorrect number of FORMAT fields at %s:%"PRId64" .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),(int64_t) line->pos+1,tok->tag,line->n_sample,tok->nsamples);
int nvals;
if ( (nvals=bcf_get_format_int32(flt->hdr,line,tok->tag,&flt->tmpi,&flt->mtmpi))<0 )
{
if ( !tok->usmpl[i] ) continue;
int32_t *ptr = flt->tmpi + i*nsrc1;
- if ( tok->idx>=nsrc1 || ptr[tok->idx]==bcf_int32_missing || ptr[tok->idx]==bcf_int32_vector_end )
+ if ( tok->idx>=nsrc1 || ptr[tok->idx]==bcf_int32_missing )
bcf_double_set_missing(tok->values[i]);
+ else if ( ptr[tok->idx]==bcf_int32_vector_end )
+ bcf_double_set_vector_end(tok->values[i]);
else
tok->values[i] = ptr[tok->idx];
}
for (k=0; k<kend; k++)
{
if ( k<tok->nidxs && !tok->idxs[k] ) continue;
- if ( src[k]==bcf_int32_missing || src[k]==bcf_int32_vector_end )
+ if ( src[k]==bcf_int32_missing )
bcf_double_set_missing(dst[j]);
+ else if ( src[k]==bcf_int32_vector_end )
+ bcf_double_set_vector_end(dst[j]);
else
dst[j] = src[k];
j++;
}
- while (j < tok->nval1)
+ if ( j==0 )
{
bcf_double_set_missing(dst[j]);
j++;
}
+ while (j < tok->nval1)
+ {
+ bcf_double_set_vector_end(dst[j]);
+ j++;
+ }
}
}
}
static void filters_set_format_float(filter_t *flt, bcf1_t *line, token_t *tok)
{
if ( line->n_sample != tok->nsamples )
- error("Incorrect number of FORMAT fields at %s:%d .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),line->pos+1,tok->tag,line->n_sample,tok->nsamples);
+ error("Incorrect number of FORMAT fields at %s:%"PRId64" .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),(int64_t) line->pos+1,tok->tag,line->n_sample,tok->nsamples);
int nvals;
if ( (nvals=bcf_get_format_float(flt->hdr,line,tok->tag,&flt->tmpf,&flt->mtmpf))<0 )
{
if ( !tok->usmpl[i] ) continue;
float *ptr = flt->tmpf + i*nsrc1;
- if ( tok->idx>=nsrc1 || bcf_float_is_missing(ptr[tok->idx]) || bcf_float_is_vector_end(ptr[tok->idx]) )
+ if ( tok->idx>=nsrc1 || bcf_float_is_missing(ptr[tok->idx]) )
bcf_double_set_missing(tok->values[i]);
+ else if ( bcf_float_is_vector_end(ptr[tok->idx]) )
+ bcf_double_set_vector_end(tok->values[i]);
else
tok->values[i] = ptr[tok->idx];
}
for (k=0; k<kend; k++)
{
if ( k<tok->nidxs && !tok->idxs[k] ) continue;
- if ( bcf_float_is_missing(src[k]) || bcf_float_is_vector_end(src[k]) )
+ if ( bcf_float_is_missing(src[k]) )
bcf_double_set_missing(dst[j]);
+ else if ( bcf_float_is_vector_end(src[k]) )
+ bcf_double_set_vector_end(dst[j]);
else
dst[j] = src[k];
j++;
}
- while (j < tok->nval1)
+ if ( j==0 )
{
bcf_double_set_missing(dst[j]);
j++;
}
+ while (j < tok->nval1)
+ {
+ bcf_double_set_vector_end(dst[j]);
+ j++;
+ }
}
}
}
static void filters_set_format_string(filter_t *flt, bcf1_t *line, token_t *tok)
{
if ( line->n_sample != tok->nsamples )
- error("Incorrect number of FORMAT fields at %s:%d .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),line->pos+1,tok->tag,line->n_sample,tok->nsamples);
+ error("Incorrect number of FORMAT fields at %s:%"PRId64" .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),(int64_t) line->pos+1,tok->tag,line->n_sample,tok->nsamples);
int i, ndim = tok->str_value.m;
int nstr = bcf_get_format_char(flt->hdr, line, tok->tag, &tok->str_value.s, &ndim);
case BCF_BT_INT8: BRANCH_INT(int8_t, bcf_int8_vector_end); break;
case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_vector_end); break;
case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_vector_end); break;
- default: error("The GT type is not lineognised: %d at %s:%d\n",fmt->type, bcf_seqname(flt->hdr,line),line->pos+1); break;
+ default: error("The GT type is not lineognised: %d at %s:%"PRId64"\n",fmt->type, bcf_seqname(flt->hdr,line),(int64_t) line->pos+1); break;
}
#undef BRANCH_INT
assert( tok->nsamples == nsmpl );
tok->nvalues = tok->str_value.l;
tok->nval1 = blen;
}
+static void filters_set_ilen(filter_t *flt, bcf1_t *line, token_t *tok)
+{
+ tok->nvalues = line->n_allele - 1;
+ hts_expand(double,tok->nvalues,tok->mvalues,tok->values);
+
+ int i, rlen = strlen(line->d.allele[0]);
+ for (i=1; i<line->n_allele; i++)
+ {
+ int alen = strlen(line->d.allele[i]);
+ if ( rlen==alen ) bcf_double_set_missing(tok->values[i-1]);
+ else tok->values[i-1] = alen - rlen;
+ }
+}
static void filters_set_ref_string(filter_t *flt, bcf1_t *line, token_t *tok)
{
tok->str_value.l = 0;
if ( rtok->pass_samples[i] ) npass++;
}
- assert( rtok->values );
- rtok->nvalues = 1;
- rtok->values[0] = rtok->tag[0]=='N' ? npass : (line->n_sample ? 1.0*npass/line->n_sample : 0);
- rtok->nsamples = 0;
+ hts_expand(double,rtok->nsamples,rtok->mvalues,rtok->values);
+ double value = rtok->tag[0]=='N' ? npass : (line->n_sample ? 1.0*npass/line->n_sample : 0);
+ rtok->nval1 = 1;
+ rtok->nvalues = rtok->nsamples;
+
+ // Set per-sample status so that `query -i 'F_PASS(GT!="mis" & GQ >= 20) > 0.5'` or +trio-stats
+ // consider only the passing site AND samples. The values for failed samples is set to -1 so
+ // that it can never conflict with valid expressions.
+ for (i=0; i<rtok->nsamples; i++)
+ rtok->values[i] = rtok->pass_samples[i] ? value : -1;
return 1;
}
int i, has_value = 0;
for (i=0; i<tok->nvalues; i++)
{
- if ( bcf_double_is_missing(tok->values[i]) || bcf_double_is_vector_end(tok->values[i]) ) continue;
+ if ( bcf_double_is_missing_or_vector_end(tok->values[i]) ) continue;
has_value = 1;
if ( val < tok->values[i] ) val = tok->values[i];
}
int i, has_value = 0;
for (i=0; i<tok->nvalues; i++)
{
- if ( bcf_double_is_missing(tok->values[i]) || bcf_double_is_vector_end(tok->values[i]) ) continue;
+ if ( bcf_double_is_missing_or_vector_end(tok->values[i]) ) continue;
has_value = 1;
if ( val > tok->values[i] ) val = tok->values[i];
}
double val = 0;
int i, n = 0;
for (i=0; i<tok->nvalues; i++)
- if ( !bcf_double_is_missing(tok->values[i]) ) { val += tok->values[i]; n++; }
+ if ( !bcf_double_is_missing_or_vector_end(tok->values[i]) ) { val += tok->values[i]; n++; }
if ( n )
{
rtok->values[0] = val / n;
}
return 1;
}
+static int compare_doubles(const void *lhs, const void *rhs)
+{
+ double arg1 = *(const double*) lhs;
+ double arg2 = *(const double*) rhs;
+ if (arg1 < arg2) return -1;
+ if (arg1 > arg2) return 1;
+ return 0;
+}
+static int func_median(filter_t *flt, bcf1_t *line, token_t *rtok, token_t **stack, int nstack)
+{
+ token_t *tok = stack[nstack - 1];
+ rtok->nvalues = 0;
+ if ( !tok->nvalues ) return 1;
+ int i, n = 0;
+ for (i=0; i<tok->nvalues; i++)
+ {
+ if ( bcf_double_is_missing_or_vector_end(tok->values[i]) ) continue;
+ if ( n < i ) tok->values[n] = tok->values[i];
+ n++;
+ }
+ if ( !n ) return 1;
+ if ( n==1 ) rtok->values[0] = tok->values[0];
+ else
+ {
+ qsort(tok->values, n, sizeof(double), compare_doubles);
+ rtok->values[0] = n % 2 ? tok->values[n/2] : (tok->values[n/2-1] + tok->values[n/2]) * 0.5;
+ }
+ rtok->nvalues = 1;
+ return 1;
+}
+static int func_stddev(filter_t *flt, bcf1_t *line, token_t *rtok, token_t **stack, int nstack)
+{
+ token_t *tok = stack[nstack - 1];
+ rtok->nvalues = 0;
+ if ( !tok->nvalues ) return 1;
+ int i, n = 0;
+ for (i=0; i<tok->nvalues; i++)
+ {
+ if ( bcf_double_is_missing_or_vector_end(tok->values[i]) ) continue;
+ if ( n < i ) tok->values[n] = tok->values[i];
+ n++;
+ }
+ if ( !n ) return 1;
+ if ( n==1 ) rtok->values[0] = 0;
+ else
+ {
+ double sdev = 0, avg = 0;
+ for (i=0; i<n; i++) avg += tok->values[n];
+ avg /= n;
+ for (i=0; i<n; i++) sdev += (tok->values[n] - avg) * (tok->values[n] - avg);
+ rtok->values[0] = sqrt(sdev/n);
+ }
+ rtok->nvalues = 1;
+ return 1;
+}
static int func_sum(filter_t *flt, bcf1_t *line, token_t *rtok, token_t **stack, int nstack)
{
rtok->nvalues = 0;
double val = 0;
int i, n = 0;
for (i=0; i<tok->nvalues; i++)
- if ( !bcf_double_is_missing(tok->values[i]) ) { val += tok->values[i]; n++; }
+ if ( !bcf_double_is_missing_or_vector_end(tok->values[i]) ) { val += tok->values[i]; n++; }
if ( n )
{
rtok->values[0] = val;
int i;
for (i=0; i<tok->nvalues; i++)
if ( bcf_double_is_missing(tok->values[i]) ) bcf_double_set_missing(rtok->values[i]);
- else rtok->values[i] = fabs(tok->values[i]);
+ else if ( !bcf_double_is_vector_end(tok->values[i]) ) rtok->values[i] = fabs(tok->values[i]);
return 1;
}
static int func_count(filter_t *flt, bcf1_t *line, token_t *rtok, token_t **stack, int nstack)
{
token_t *tok = stack[nstack - 1];
- if ( !tok->nsamples ) error("COUNT() can be applied only on FORMAT fields\n");
-
int i, cnt = 0;
- for (i=0; i<tok->nsamples; i++)
- if ( tok->pass_samples[i] ) cnt++;
+ if ( !tok->nsamples )
+ {
+ if ( tok->is_str )
+ {
+ if ( tok->str_value.l ) cnt = 1;
+ for (i=0; i<tok->str_value.l; i++) if ( tok->str_value.s[i]==',' ) cnt++;
+ }
+ else
+ cnt = tok->nvalues;
+ }
+ else
+ {
+ for (i=0; i<tok->nsamples; i++)
+ if ( tok->pass_samples[i] ) cnt++;
+ }
rtok->nvalues = 1;
rtok->values[0] = cnt;
}
int idx1 = bcf_gt_allele(ptr[0]);
int idx2 = bcf_gt_allele(ptr[1]);
- if ( idx1>=line->n_allele ) error("Incorrect allele index at %s:%d, sample %s\n", bcf_seqname(flt->hdr,line),line->pos+1,flt->hdr->samples[i]);
- if ( idx2>=line->n_allele ) error("Incorrect allele index at %s:%d, sample %s\n", bcf_seqname(flt->hdr,line),line->pos+1,flt->hdr->samples[i]);
+ if ( idx1>=line->n_allele ) error("Incorrect allele index at %s:%"PRId64", sample %s\n", bcf_seqname(flt->hdr,line),(int64_t) line->pos+1,flt->hdr->samples[i]);
+ if ( idx2>=line->n_allele ) error("Incorrect allele index at %s:%"PRId64", sample %s\n", bcf_seqname(flt->hdr,line),(int64_t) line->pos+1,flt->hdr->samples[i]);
double *vals = tok->values + tok->nval1*i;
- if ( bcf_double_is_missing(vals[idx1]) || bcf_double_is_missing(vals[idx2]) )
+ if ( bcf_double_is_missing_or_vector_end(vals[idx1]) || bcf_double_is_missing_or_vector_end(vals[idx2]) )
{
bcf_double_set_missing(rtok->values[i]);
continue;
// the fields given explicitly: binom(AD[:0],AD[:1])
token_t *tok2 = stack[istack+1];
if ( tok->nval1!=1 || tok2->nval1!=1 )
- error("Expected one value per binom() argument, found %d and %d at %s:%d\n",tok->nval1,tok2->nval1, bcf_seqname(flt->hdr,line),line->pos+1);
+ error("Expected one value per binom() argument, found %d and %d at %s:%"PRId64"\n",tok->nval1,tok2->nval1, bcf_seqname(flt->hdr,line),(int64_t) line->pos+1);
for (i=0; i<rtok->nsamples; i++)
{
if ( !rtok->usmpl[i] ) continue;
double *ptr1 = tok->values + tok->nval1*i;
double *ptr2 = tok2->values + tok2->nval1*i;
- if ( bcf_double_is_missing(ptr1[0]) || bcf_double_is_missing(ptr2[0]) )
+ if ( bcf_double_is_missing_or_vector_end(ptr1[0]) || bcf_double_is_missing_or_vector_end(ptr2[0]) )
{
bcf_double_set_missing(rtok->values[i]);
continue;
ptr2 = &tok2->values[0];
}
}
- if ( !ptr1 || !ptr2 || bcf_double_is_missing(ptr1[0]) || bcf_double_is_missing(ptr2[0]) )
+ if ( !ptr1 || !ptr2 || bcf_double_is_missing_or_vector_end(ptr1[0]) || bcf_double_is_missing_or_vector_end(ptr2[0]) )
bcf_double_set_missing(rtok->values[0]);
else
{
}
return rtok->nargs;
}
+static int func_phred(filter_t *flt, bcf1_t *line, token_t *rtok, token_t **stack, int nstack)
+{
+ token_t *tok = stack[nstack - 1];
+ if ( tok->is_str ) error("PHRED() can be applied only on numeric values\n");
+
+ rtok->nsamples = tok->nsamples;
+ rtok->nval1 = tok->nval1;
+ memcpy(rtok->pass_samples, tok->pass_samples, rtok->nsamples*sizeof(*rtok->pass_samples));
+ assert(tok->usmpl);
+ if ( !rtok->usmpl )
+ {
+ rtok->usmpl = (uint8_t*) malloc(tok->nsamples*sizeof(*rtok->usmpl));
+ memcpy(rtok->usmpl, tok->usmpl, tok->nsamples*sizeof(*rtok->usmpl));
+ }
+ rtok->nvalues = tok->nvalues;
+ if ( !tok->nvalues ) return 1;
+
+ hts_expand(double, rtok->nvalues, rtok->mvalues, rtok->values);
+ int i;
+ for (i=0; i<tok->nvalues; i++)
+ if ( bcf_double_is_missing_or_vector_end(tok->values[i]) ) bcf_double_set_missing(rtok->values[i]);
+ else rtok->values[i] = -4.34294481903*log(tok->values[i]);
+
+ return 1;
+}
inline static void tok_init_values(token_t *atok, token_t *btok, token_t *rtok)
{
token_t *tok = atok->nvalues > btok->nvalues ? atok : btok;
assert( atok->nsamples==btok->nsamples ); \
for (i=0; i<atok->nvalues; i++) \
{ \
- if ( bcf_double_is_missing(atok->values[i]) || bcf_double_is_missing(btok->values[i]) ) \
+ if ( bcf_double_is_missing_or_vector_end(atok->values[i]) || bcf_double_is_missing_or_vector_end(btok->values[i]) ) \
{ \
bcf_double_set_missing(rtok->values[i]); \
continue; \
token_t *xtok = atok->nsamples ? atok : btok; \
token_t *ytok = atok->nsamples ? btok : atok; \
assert( ytok->nvalues==1 ); \
- if ( !bcf_double_is_missing(ytok->values[0]) ) \
+ if ( !bcf_double_is_missing_or_vector_end(ytok->values[0]) ) \
{ \
for (i=0; i<xtok->nvalues; i++) \
{ \
- if ( bcf_double_is_missing(xtok->values[i]) ) \
+ if ( bcf_double_is_missing_or_vector_end(xtok->values[i]) ) \
{ \
bcf_double_set_missing(rtok->values[i]); \
continue; \
{ \
token_t *rtok = _rtok; \
int i, j, k; \
- assert( !atok->nsamples || !btok->nsamples ); \
tok_init_samples(atok, btok, rtok); \
if ( !atok->nsamples && !btok->nsamples ) \
{ \
token_t *tok = atok->nvalues ? atok : btok; \
for (j=0; j<tok->nvalues; j++) \
{ \
- if ( bcf_double_is_missing(tok->values[j]) ) \
+ if ( bcf_double_is_missing_or_vector_end(tok->values[j]) ) \
{ \
if ( missing_logic[2] ) { rtok->pass_site = 1; break; } \
} \
{ \
for (i=0; i<atok->nvalues; i++) \
{ \
- int amiss = bcf_double_is_missing(atok->values[i]) ? 1 : 0; \
+ int amiss = bcf_double_is_missing_or_vector_end(atok->values[i]) ? 1 : 0; \
for (j=0; j<btok->nvalues; j++) \
{ \
- int nmiss = amiss + (bcf_double_is_missing(btok->values[j]) ? 1 : 0); \
+ int nmiss = amiss + (bcf_double_is_missing_or_vector_end(btok->values[j]) ? 1 : 0); \
if ( nmiss ) \
{ \
if ( missing_logic[nmiss] ) { rtok->pass_site = 1; i = atok->nvalues; break; } \
} \
- else if ( atok->values[i] CMP_OP btok->values[j] ) { rtok->pass_site = 1; i = atok->nvalues; break; } \
+ else if ( atok->values[i] > 16777216 || btok->values[j] > 16777216 ) /* Ugly, see #871 */ \
+ { \
+ if ( atok->values[i] CMP_OP btok->values[j] ) { rtok->pass_site = 1; i = atok->nvalues; break; } \
+ } \
+ else if ( (float)atok->values[i] CMP_OP (float)btok->values[j] ) { rtok->pass_site = 1; i = atok->nvalues; break; } \
} \
} \
} \
{ \
int miss = 0; \
for (j=0; j<tok->nvalues; j++) \
- miss |= bcf_double_is_missing(tok->values[j]) ? 1 : 0; \
+ miss |= bcf_double_is_missing_or_vector_end(tok->values[j]) ? 1 : 0; \
if ( missing_logic[++miss] ) \
{ \
for (i=0; i<rtok->nsamples; i++) \
double *ptr = tok->values + i*tok->nval1; \
int miss = 0; \
for (j=0; j<tok->nval1; j++) \
- miss |= bcf_double_is_missing(ptr[j]) ? 1 : 0; \
+ miss |= bcf_double_is_missing_or_vector_end(ptr[j]) ? 1 : 0; \
if ( missing_logic[++miss] ) { rtok->pass_samples[i] = missing_logic[miss]; rtok->pass_site = 1; } \
} \
} \
+ else if ( atok->nsamples && btok->nsamples ) \
+ { \
+ if ( atok->nval1!=btok->nval1 ) error("Incompatible number of per-sample values in comparison: %d vs %d\n",atok->nval1,btok->nval1); \
+ if ( atok->nsamples!=btok->nsamples ) error("Incompatible number samples in comparison: %d vs %d\n",atok->nsamples,btok->nsamples); \
+ for (i=0; i<atok->nsamples; i++) \
+ { \
+ if ( !atok->usmpl[i] || !btok->usmpl[i] ) { rtok->usmpl[i] = 0; continue; } \
+ double *aptr = atok->values + i*atok->nval1; \
+ double *bptr = btok->values + i*btok->nval1; \
+ for (j=0; j<atok->nval1; j++) \
+ { \
+ int nmiss = bcf_double_is_missing_or_vector_end(aptr[j]) ? 1 : 0; \
+ if ( nmiss && !missing_logic[0] ) continue; /* any is missing => result is false */ \
+ nmiss += (bcf_double_is_missing_or_vector_end(bptr[j]) ? 1 : 0); \
+ if ( nmiss ) \
+ { \
+ if ( missing_logic[nmiss] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; break; } \
+ } \
+ else if ( aptr[j] > 16777216 || bptr[j] > 16777216 ) /* Ugly, see #871 */ \
+ { \
+ if ( aptr[j] CMP_OP bptr[j] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; break; } \
+ } \
+ else if ( (float)aptr[j] CMP_OP (float)bptr[j] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; break; } \
+ } \
+ } \
+ } \
else \
{ \
token_t *xtok = atok->nsamples ? atok : btok; \
double *yptr = ytok->values + i*ytok->nval1; \
for (j=0; j<xtok->nval1; j++) \
{ \
- int miss = bcf_double_is_missing(xptr[j]) ? 1 : 0; \
+ int miss = bcf_double_is_missing_or_vector_end(xptr[j]) ? 1 : 0; \
if ( miss && !missing_logic[0] ) continue; /* any is missing => result is false */ \
for (k=0; k<ytok->nvalues; k++) \
{ \
- int nmiss = miss + (bcf_double_is_missing(yptr[k]) ? 1 : 0); \
+ int nmiss = miss + (bcf_double_is_missing_or_vector_end(yptr[k]) ? 1 : 0); \
if ( nmiss ) \
{ \
if ( missing_logic[nmiss] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; j = xtok->nval1; break; } \
} \
- else if ( xptr[j] CMP_OP yptr[k] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; j = xtok->nval1; break; } \
+ else if ( xptr[j] > 16777216 || yptr[k] > 16777216 ) /* Ugly, see #871 */ \
+ { \
+ if ( xptr[j] CMP_OP yptr[k] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; j = xtok->nval1; break; } \
+ } \
+ else if ( (float)xptr[j] CMP_OP (float)yptr[k] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; j = xtok->nval1; break; } \
} \
} \
} \
int *idxs2 = NULL, nidxs2 = 0, idx2 = 0;
int set_samples = 0;
- char *colon = rindex(tag_idx, ':');
+ char *colon = strrchr(tag_idx, ':');
if ( tag_idx[0]=='@' ) // file list with sample names
{
if ( !is_fmt ) error("Could not parse \"%s\". (Not a FORMAT tag yet a sample list provided.)\n", ori);
char *fname = expand_path(tag_idx+1);
+#ifdef _WIN32
+ if (fname && strlen(fname) > 2 && fname[1] == ':') // Deal with Windows paths, such as 'C:\..'
+ colon = strrchr(fname+2, ':');
+#endif
int nsmpl;
char **list = hts_readlist(fname, 1, &nsmpl);
if ( !list && colon )
tok->idxs = idxs2;
tok->nidxs = nidxs2;
tok->idx = idx2;
- colon = rindex(fname, ':');
+ colon = strrchr(fname, ':');
*colon = 0;
list = hts_readlist(fname, 1, &nsmpl);
}
}
static int filters_init1(filter_t *filter, char *str, int len, token_t *tok)
{
+ tok->tag_type = -1;
tok->tok_type = TOK_VAL;
tok->hdr_id = -1;
tok->pass_site = -1;
tok->comparator = filters_cmp_filter;
tok->tag = strdup("FILTER");
filter->max_unpack |= BCF_UN_FLT;
+ tok->tag_type = BCF_HL_FLT;
return 0;
}
else if ( !strncasecmp(str,"ID",len) || !strncasecmp(str,"%ID",len) /* for backward compatibility */ )
tok->tag = strdup("ID");
return 0;
}
+ else if ( !strncasecmp(str,"CHROM",len) )
+ {
+ tok->setter = &filters_set_chrom;
+ tok->tag = strdup("CHROM");
+ return 0;
+ }
else if ( !strncasecmp(str,"POS",len) )
{
tok->setter = &filters_set_pos;
}
else if ( !strncasecmp(str,"N_MISSING",len) )
{
+ filter->max_unpack |= BCF_UN_FMT;
tok->setter = &filters_set_nmissing;
tok->tag = strdup("N_MISSING");
return 0;
}
else if ( !strncasecmp(str,"F_MISSING",len) )
{
+ filter->max_unpack |= BCF_UN_FMT;
tok->setter = &filters_set_nmissing;
tok->tag = strdup("F_MISSING");
return 0;
for (i=0; i<tok->nsamples; i++) tok->usmpl[i] = 1;
}
- tok->type = is_fmt ? BCF_HL_FMT : BCF_HL_INFO;
+ tok->tag_type = is_fmt ? BCF_HL_FMT : BCF_HL_INFO;
if ( is_fmt ) filter->max_unpack |= BCF_UN_FMT;
if ( tok->hdr_id>=0 )
{
free(tmp.s);
return 0;
}
+ else if ( !strcasecmp(tmp.s,"ILEN") || !strcasecmp(tmp.s,"%ILEN") )
+ {
+ filter->max_unpack |= BCF_UN_STR;
+ tok->setter = &filters_set_ilen;
+ tok->tag = strdup("ILEN");
+ free(tmp.s);
+ return 0;
+ }
// is it a value? Here we parse as integer/float separately and use strtof
// rather than strtod, because the more accurate double representation
// would invalidate floating point comparisons like QUAL=59.2, obtained via
- // htslib/vcf parser
+ // htslib/vcf parser.
+ // Update: use strtod() and force floats only in comparisons
char *end;
tok->threshold = strtol(tmp.s, &end, 10); // integer?
if ( end - tmp.s != strlen(tmp.s) )
{
errno = 0;
- tok->threshold = strtof(tmp.s, &end); // float?
+ tok->threshold = strtod(tmp.s, &end); // float?
if ( errno!=0 || end!=tmp.s+len ) error("[%s:%d %s] Error: the tag \"%s\" is not defined in the VCF header\n", __FILE__,__LINE__,__FUNCTION__,tmp.s);
}
tok->is_constant = 1;
if ( ret==-1 ) error("Missing quotes in: %s\n", str);
// fprintf(stderr,"token=[%c] .. [%s] %d\n", TOKEN_STRING[ret], tmp, len);
- // int i; for (i=0; i<nops; i++) fprintf(stderr," .%c", TOKEN_STRING[ops[i]]); fprintf(stderr,"\n");
+ // int i; for (i=0; i<nops; i++) fprintf(stderr," .%c", TOKEN_STRING[ops[i].tok_type]); fprintf(stderr,"\n");
if ( ret==TOK_LFT ) // left bracket
{
tok->hdr_id = -1;
tok->pass_site = -1;
tok->threshold = -1.0;
- if ( !strncasecmp(tmp-len,"N_PASS",6) ) { tok->func = func_npass; tok->tag = strdup("N_PASS"); }
- else if ( !strncasecmp(tmp-len,"F_PASS",6) ) { tok->func = func_npass; tok->tag = strdup("F_PASS"); }
+ if ( !strncasecmp(tmp-len,"N_PASS",6) )
+ {
+ filter->max_unpack |= BCF_UN_FMT;
+ tok->func = func_npass;
+ tok->tag = strdup("N_PASS");
+ }
+ else if ( !strncasecmp(tmp-len,"F_PASS",6) )
+ {
+ filter->max_unpack |= BCF_UN_FMT;
+ tok->func = func_npass;
+ tok->tag = strdup("F_PASS");
+ }
else error("The function \"%s\" is not supported\n", tmp-len);
continue;
}
// list of operators and convert the strings (e.g. "PASS") to BCF ids. The string value token must be
// just before or after the FILTER token and they must be followed with a comparison operator.
// At this point we also initialize regex expressions which, in RPN, must preceed the LIKE/NLIKE operator.
- // Additionally, treat "." as missing value rather than a string in numeric equalities.
+ // Additionally, treat "." as missing value rather than a string in numeric equalities; that
+ // @file is only used with ID; etc.
// This code is fragile: improve me.
int i;
for (i=0; i<nout; i++)
if ( i+1<nout && (out[i].tok_type==TOK_LT || out[i].tok_type==TOK_BT) && out[i+1].tok_type==TOK_EQ )
error("Error parsing the expression: \"%s\"\n", filter->str);
+ if ( out[i].hash )
+ {
+ int j = out[i+1].tok_type==TOK_VAL ? i+1 : i-1;
+ if ( out[j].comparator!=filters_cmp_id )
+ error("Error: could not parse the expression. Note that the \"@file_name\" syntax can be currently used with ID column only.\n");
+ }
if ( out[i].tok_type==TOK_OR || out[i].tok_type==TOK_OR_VEC )
out[i].func = vector_logic_or;
if ( out[i].tok_type==TOK_AND || out[i].tok_type==TOK_AND_VEC )
int set_missing = 0;
if ( out[k].hdr_id>0 )
{
- int type = bcf_hdr_id2type(filter->hdr,out[k].type,out[k].hdr_id);
+ int type = bcf_hdr_id2type(filter->hdr,out[k].tag_type,out[k].hdr_id);
if ( type==BCF_HT_INT ) set_missing = 1;
else if ( type==BCF_HT_REAL ) set_missing = 1;
}
}
if ( out[i].tok_type!=TOK_VAL ) continue;
if ( !out[i].tag ) continue;
- if ( !strcmp(out[i].tag,"TYPE") )
+ if ( out[i].setter==filters_set_type )
{
if ( i+1==nout ) error("Could not parse the expression: %s\n", filter->str);
int itok, ival;
else if ( !strcasecmp(out[ival].key,"mnp") || !strcasecmp(out[ival].key,"mnps") ) { out[ival].threshold = VCF_MNP<<1; out[ival].is_str = 0; }
else if ( !strcasecmp(out[ival].key,"other") ) { out[ival].threshold = VCF_OTHER<<1; out[ival].is_str = 0; }
else if ( !strcasecmp(out[ival].key,"bnd") ) { out[ival].threshold = VCF_BND<<1; out[ival].is_str = 0; }
+ else if ( !strcasecmp(out[ival].key,"overlap") ) { out[ival].threshold = VCF_OVERLAP<<1; out[ival].is_str = 0; }
else if ( !strcasecmp(out[ival].key,"ref") ) { out[ival].threshold = 1; out[ival].is_str = 0; }
else error("The type \"%s\" not recognised: %s\n", out[ival].key, filter->str);
if ( out[itok].tok_type==TOK_LIKE || out[itok].tok_type==TOK_NLIKE ) out[itok].comparator = filters_cmp_bit_and;
else if ( !strcasecmp(out[ival].key,"r") ) { out[i].setter = filters_set_genotype2; out[ival].key[0]='r'; out[ival].key[1]=0; } // r
continue;
}
- if ( !strcmp(out[i].tag,"FILTER") )
+ if ( out[i].tag_type==BCF_HL_FLT )
{
if ( i+1==nout ) error("Could not parse the expression: %s\n", filter->str);
int itok = i, ival;
filter->nsamples = filter->max_unpack&BCF_UN_FMT ? bcf_hdr_nsamples(filter->hdr) : 0;
for (i=0; i<nout; i++)
{
- if ( out[i].tok_type==TOK_MAX ) { out[i].func = func_max; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_MIN ) { out[i].func = func_min; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_AVG ) { out[i].func = func_avg; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_SUM ) { out[i].func = func_sum; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_ABS ) { out[i].func = func_abs; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_CNT ) { out[i].func = func_count; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_LEN ) { out[i].func = func_strlen; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
+ if ( out[i].tok_type==TOK_MAX ) { out[i].func = func_max; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_MIN ) { out[i].func = func_min; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_AVG ) { out[i].func = func_avg; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_MEDIAN ) { out[i].func = func_median; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_AVG ) { out[i].func = func_avg; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_STDEV ) { out[i].func = func_stddev; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_SUM ) { out[i].func = func_sum; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_ABS ) { out[i].func = func_abs; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_CNT ) { out[i].func = func_count; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_LEN ) { out[i].func = func_strlen; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_PHRED ) { out[i].func = func_phred; out[i].tok_type = TOK_FUNC; }
else if ( out[i].tok_type==TOK_BINOM ) { out[i].func = func_binom; out[i].tok_type = TOK_FUNC; }
else if ( out[i].tok_type==TOK_PERLSUB ) { out[i].func = perl_exec; out[i].tok_type = TOK_FUNC; }
hts_expand0(double,1,out[i].mvalues,out[i].values);
}
else
{
+ if ( is_str==1 ) error("Error: cannot use arithmetic operators to compare strings and numbers\n");
+
// Determine what to do with one [1] or both [2] sides missing. The first field [0] gives [1]|[2]
int missing_logic[] = {0,0,0};
if ( filter->filters[i].tok_type == TOK_EQ ) { missing_logic[0] = missing_logic[2] = 1; }
CMP_VECTORS(filter->flt_stack[nstack-2],filter->flt_stack[nstack-1],&filter->filters[i],>=,missing_logic)
else
error("todo: %s:%d .. type=%d\n", __FILE__,__LINE__,filter->filters[i].tok_type);
-
}
filter->flt_stack[nstack-2] = &filter->filters[i];
nstack--;
#include <errno.h>
#include <math.h>
#include <sys/types.h>
+#include <inttypes.h>
+#ifndef _WIN32
#include <pwd.h>
+#endif
#include <regex.h>
#include <htslib/khash_str2int.h>
#include <htslib/hts_defs.h>
# define __FUNCTION__ __func__
#endif
-uint64_t bcf_double_missing = 0x7ff0000000000001;
-uint64_t bcf_double_vector_end = 0x7ff0000000000002;
+static const uint64_t bcf_double_missing = 0x7ff0000000000001;
+static const uint64_t bcf_double_vector_end = 0x7ff0000000000002;
static inline void bcf_double_set(double *ptr, uint64_t value)
{
union { uint64_t i; double d; } u;
#define bcf_double_set_missing(x) bcf_double_set(&(x),bcf_double_missing)
#define bcf_double_is_vector_end(x) bcf_double_test((x),bcf_double_vector_end)
#define bcf_double_is_missing(x) bcf_double_test((x),bcf_double_missing)
+#define bcf_double_is_missing_or_vector_end(x) (bcf_double_test((x),bcf_double_missing) || bcf_double_test((x),bcf_double_vector_end))
typedef struct _token_t
char *tag; // for debugging and printout only, VCF tag name
double threshold; // filtering threshold
int is_constant; // the threshold is set
- int hdr_id, type; // BCF header lookup ID and one of BCF_HT_* types
+ int hdr_id, tag_type; // BCF header lookup ID and one of BCF_HL_* types
int idx; // 0-based index to VCF vectors,
// -2: list (e.g. [0,1,2] or [1..3] or [1..] or any field[*], which is equivalent to [0..])
int *idxs; // set indexes to 0 to exclude, to 1 to include, and last element negative if unlimited
#define TOK_CNT 26
#define TOK_PERLSUB 27
#define TOK_BINOM 28
+#define TOK_PHRED 29
+#define TOK_MEDIAN 30
+#define TOK_STDEV 31
-// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
-// ( ) [ < = > ] ! | & + - * / M m a A O ~ ^ S . l f c p
-static int op_prec[] = {0,1,1,5,5,5,5,5,5,2,3, 6, 6, 7, 7, 8, 8, 8, 3, 2, 5, 5, 8, 8, 8, 8, 8, 8};
-#define TOKEN_STRING "x()[<=>]!|&+-*/MmaAO~^S.lfcp"
+// 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
+// ( ) [ < = > ] ! | & + - * / M m a A O ~ ^ S . l f c p b P i s
+static int op_prec[] = {0,1,1,5,5,5,5,5,5,2,3, 6, 6, 7, 7, 8, 8, 8, 3, 2, 5, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8};
+#define TOKEN_STRING "x()[<=>]!|&+-*/MmaAO~^S.lfcpis"
// Return negative values if it is a function with variable number of arguments
static int filters_next_token(char **str, int *len)
if ( !strncasecmp(tmp,"MAX(",4) ) { (*str) += 3; return TOK_MAX; }
if ( !strncasecmp(tmp,"MIN(",4) ) { (*str) += 3; return TOK_MIN; }
+ if ( !strncasecmp(tmp,"MEAN(",5) ) { (*str) += 4; return TOK_AVG; }
+ if ( !strncasecmp(tmp,"MEDIAN(",7) ) { (*str) += 6; return TOK_MEDIAN; }
if ( !strncasecmp(tmp,"AVG(",4) ) { (*str) += 3; return TOK_AVG; }
+ if ( !strncasecmp(tmp,"STDEV(",6) ) { (*str) += 5; return TOK_STDEV; }
if ( !strncasecmp(tmp,"SUM(",4) ) { (*str) += 3; return TOK_SUM; }
if ( !strncasecmp(tmp,"ABS(",4) ) { (*str) += 3; return TOK_ABS; }
if ( !strncasecmp(tmp,"COUNT(",4) ) { (*str) += 5; return TOK_CNT; }
if ( !strncasecmp(tmp,"STRLEN(",7) ) { (*str) += 6; return TOK_LEN; }
if ( !strncasecmp(tmp,"BINOM(",6) ) { (*str) += 5; return -TOK_BINOM; }
+ if ( !strncasecmp(tmp,"PHRED(",6) ) { (*str) += 5; return TOK_PHRED; }
if ( !strncasecmp(tmp,"%MAX(",5) ) { (*str) += 4; return TOK_MAX; } // for backward compatibility
if ( !strncasecmp(tmp,"%MIN(",5) ) { (*str) += 4; return TOK_MIN; } // for backward compatibility
if ( !strncasecmp(tmp,"%AVG(",5) ) { (*str) += 4; return TOK_AVG; } // for backward compatibility
if ( !strncasecmp(tmp,"PERL.",5) ) { (*str) += 5; return -TOK_PERLSUB; }
if ( !strncasecmp(tmp,"N_PASS(",7) ) { *len = 6; (*str) += 6; return -TOK_FUNC; }
if ( !strncasecmp(tmp,"F_PASS(",7) ) { *len = 6; (*str) += 6; return -TOK_FUNC; }
+ if ( !strncasecmp(tmp,"%ILEN",5) ) { *len = 5; return TOK_VAL; } // to be able to distinguish between INFO/ILEN and on-the-fly ILEN
if ( tmp[0]=='@' ) // file name
{
}
-/*
+/*
Simple path expansion, expands ~/, ~user, $var. The result must be freed by the caller.
- Based on jkb's staden code with some adjustements.
+ Based on jkb's staden code with some adjustments.
https://sourceforge.net/p/staden/code/HEAD/tree/staden/trunk/src/Misc/getfile.c#l123
*/
char *expand_path(char *path)
{
-#ifdef _WIN32
- return strdup(path); // windows expansion: todo
-#endif
-
kstring_t str = {0,0,0};
if ( path[0] == '~' )
{
if ( !path[1] || path[1] == '/' )
{
+#ifdef _WIN32
+ kputs(getenv("HOMEDRIVE"), &str);
+ kputs(getenv("HOMEPATH"), &str);
+#else
// ~ or ~/path
kputs(getenv("HOME"), &str);
if ( path[1] ) kputs(path+1, &str);
+#endif
}
+#ifndef _WIN32
else
{
// user name: ~pd3/path
else kputs(pwentry->pw_dir, &str);
kputs(end, &str);
}
- return str.s;
+#endif
+ return ks_release(&str);
}
if ( path[0] == '$' )
{
char *var = getenv(path+1);
- if ( var ) path = var;
+ if ( var ) {
+ kputs(var, &str);
+ return ks_release(&str);
+ }
}
+
return strdup(path);
}
return;
}
+ if ( !btok->str_value.l ) error("Error occurred while evaluating the expression\n");
+
if ( rtok->tok_type==TOK_EQ )
rtok->pass_site = strcmp(btok->str_value.s,line->d.id) ? 0 : 1;
else
return -1; // this shouldn't happen
}
+static void filters_set_chrom(filter_t *flt, bcf1_t *line, token_t *tok)
+{
+ tok->str_value.l = 0;
+ kputs(bcf_seqname(flt->hdr,line), &tok->str_value);
+ tok->nvalues = tok->str_value.l;
+ tok->is_str = 1;
+}
+
static void filters_set_pos(filter_t *flt, bcf1_t *line, token_t *tok)
{
tok->values[0] = line->pos+1;
static void filters_set_format_int(filter_t *flt, bcf1_t *line, token_t *tok)
{
if ( line->n_sample != tok->nsamples )
- error("Incorrect number of FORMAT fields at %s:%d .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),line->pos+1,tok->tag,line->n_sample,tok->nsamples);
+ error("Incorrect number of FORMAT fields at %s:%"PRId64" .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),(int64_t) line->pos+1,tok->tag,line->n_sample,tok->nsamples);
int nvals;
if ( (nvals=bcf_get_format_int32(flt->hdr,line,tok->tag,&flt->tmpi,&flt->mtmpi))<0 )
{
if ( !tok->usmpl[i] ) continue;
int32_t *ptr = flt->tmpi + i*nsrc1;
- if ( tok->idx>=nsrc1 || ptr[tok->idx]==bcf_int32_missing || ptr[tok->idx]==bcf_int32_vector_end )
+ if ( tok->idx>=nsrc1 || ptr[tok->idx]==bcf_int32_missing )
bcf_double_set_missing(tok->values[i]);
+ else if ( ptr[tok->idx]==bcf_int32_vector_end )
+ bcf_double_set_vector_end(tok->values[i]);
else
tok->values[i] = ptr[tok->idx];
}
for (k=0; k<kend; k++)
{
if ( k<tok->nidxs && !tok->idxs[k] ) continue;
- if ( src[k]==bcf_int32_missing || src[k]==bcf_int32_vector_end )
+ if ( src[k]==bcf_int32_missing )
bcf_double_set_missing(dst[j]);
+ else if ( src[k]==bcf_int32_vector_end )
+ bcf_double_set_vector_end(dst[j]);
else
dst[j] = src[k];
j++;
}
- while (j < tok->nval1)
+ if ( j==0 )
{
bcf_double_set_missing(dst[j]);
j++;
}
+ while (j < tok->nval1)
+ {
+ bcf_double_set_vector_end(dst[j]);
+ j++;
+ }
}
}
}
static void filters_set_format_float(filter_t *flt, bcf1_t *line, token_t *tok)
{
if ( line->n_sample != tok->nsamples )
- error("Incorrect number of FORMAT fields at %s:%d .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),line->pos+1,tok->tag,line->n_sample,tok->nsamples);
+ error("Incorrect number of FORMAT fields at %s:%"PRId64" .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),(int64_t) line->pos+1,tok->tag,line->n_sample,tok->nsamples);
int nvals;
if ( (nvals=bcf_get_format_float(flt->hdr,line,tok->tag,&flt->tmpf,&flt->mtmpf))<0 )
{
if ( !tok->usmpl[i] ) continue;
float *ptr = flt->tmpf + i*nsrc1;
- if ( tok->idx>=nsrc1 || bcf_float_is_missing(ptr[tok->idx]) || bcf_float_is_vector_end(ptr[tok->idx]) )
+ if ( tok->idx>=nsrc1 || bcf_float_is_missing(ptr[tok->idx]) )
bcf_double_set_missing(tok->values[i]);
+ else if ( bcf_float_is_vector_end(ptr[tok->idx]) )
+ bcf_double_set_vector_end(tok->values[i]);
else
tok->values[i] = ptr[tok->idx];
}
for (k=0; k<kend; k++)
{
if ( k<tok->nidxs && !tok->idxs[k] ) continue;
- if ( bcf_float_is_missing(src[k]) || bcf_float_is_vector_end(src[k]) )
+ if ( bcf_float_is_missing(src[k]) )
bcf_double_set_missing(dst[j]);
+ else if ( bcf_float_is_vector_end(src[k]) )
+ bcf_double_set_vector_end(dst[j]);
else
dst[j] = src[k];
j++;
}
- while (j < tok->nval1)
+ if ( j==0 )
{
bcf_double_set_missing(dst[j]);
j++;
}
+ while (j < tok->nval1)
+ {
+ bcf_double_set_vector_end(dst[j]);
+ j++;
+ }
}
}
}
static void filters_set_format_string(filter_t *flt, bcf1_t *line, token_t *tok)
{
if ( line->n_sample != tok->nsamples )
- error("Incorrect number of FORMAT fields at %s:%d .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),line->pos+1,tok->tag,line->n_sample,tok->nsamples);
+ error("Incorrect number of FORMAT fields at %s:%"PRId64" .. %s, %d vs %d\n", bcf_seqname(flt->hdr,line),(int64_t) line->pos+1,tok->tag,line->n_sample,tok->nsamples);
int i, ndim = tok->str_value.m;
int nstr = bcf_get_format_char(flt->hdr, line, tok->tag, &tok->str_value.s, &ndim);
case BCF_BT_INT8: BRANCH_INT(int8_t, bcf_int8_vector_end); break;
case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_vector_end); break;
case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_vector_end); break;
- default: error("The GT type is not lineognised: %d at %s:%d\n",fmt->type, bcf_seqname(flt->hdr,line),line->pos+1); break;
+ default: error("The GT type is not lineognised: %d at %s:%"PRId64"\n",fmt->type, bcf_seqname(flt->hdr,line),(int64_t) line->pos+1); break;
}
#undef BRANCH_INT
assert( tok->nsamples == nsmpl );
tok->nvalues = tok->str_value.l;
tok->nval1 = blen;
}
+static void filters_set_ilen(filter_t *flt, bcf1_t *line, token_t *tok)
+{
+ tok->nvalues = line->n_allele - 1;
+ hts_expand(double,tok->nvalues,tok->mvalues,tok->values);
+
+ int i, rlen = strlen(line->d.allele[0]);
+ for (i=1; i<line->n_allele; i++)
+ {
+ int alen = strlen(line->d.allele[i]);
+ if ( rlen==alen ) bcf_double_set_missing(tok->values[i-1]);
+ else tok->values[i-1] = alen - rlen;
+ }
+}
static void filters_set_ref_string(filter_t *flt, bcf1_t *line, token_t *tok)
{
tok->str_value.l = 0;
if ( rtok->pass_samples[i] ) npass++;
}
- assert( rtok->values );
- rtok->nvalues = 1;
- rtok->values[0] = rtok->tag[0]=='N' ? npass : (line->n_sample ? 1.0*npass/line->n_sample : 0);
- rtok->nsamples = 0;
+ hts_expand(double,rtok->nsamples,rtok->mvalues,rtok->values);
+ double value = rtok->tag[0]=='N' ? npass : (line->n_sample ? 1.0*npass/line->n_sample : 0);
+ rtok->nval1 = 1;
+ rtok->nvalues = rtok->nsamples;
+
+ // Set per-sample status so that `query -i 'F_PASS(GT!="mis" & GQ >= 20) > 0.5'` or +trio-stats
+ // consider only the passing site AND samples. The values for failed samples is set to -1 so
+ // that it can never conflict with valid expressions.
+ for (i=0; i<rtok->nsamples; i++)
+ rtok->values[i] = rtok->pass_samples[i] ? value : -1;
return 1;
}
int i, has_value = 0;
for (i=0; i<tok->nvalues; i++)
{
- if ( bcf_double_is_missing(tok->values[i]) || bcf_double_is_vector_end(tok->values[i]) ) continue;
+ if ( bcf_double_is_missing_or_vector_end(tok->values[i]) ) continue;
has_value = 1;
if ( val < tok->values[i] ) val = tok->values[i];
}
int i, has_value = 0;
for (i=0; i<tok->nvalues; i++)
{
- if ( bcf_double_is_missing(tok->values[i]) || bcf_double_is_vector_end(tok->values[i]) ) continue;
+ if ( bcf_double_is_missing_or_vector_end(tok->values[i]) ) continue;
has_value = 1;
if ( val > tok->values[i] ) val = tok->values[i];
}
double val = 0;
int i, n = 0;
for (i=0; i<tok->nvalues; i++)
- if ( !bcf_double_is_missing(tok->values[i]) ) { val += tok->values[i]; n++; }
+ if ( !bcf_double_is_missing_or_vector_end(tok->values[i]) ) { val += tok->values[i]; n++; }
if ( n )
{
rtok->values[0] = val / n;
}
return 1;
}
+static int compare_doubles(const void *lhs, const void *rhs)
+{
+ double arg1 = *(const double*) lhs;
+ double arg2 = *(const double*) rhs;
+ if (arg1 < arg2) return -1;
+ if (arg1 > arg2) return 1;
+ return 0;
+}
+static int func_median(filter_t *flt, bcf1_t *line, token_t *rtok, token_t **stack, int nstack)
+{
+ token_t *tok = stack[nstack - 1];
+ rtok->nvalues = 0;
+ if ( !tok->nvalues ) return 1;
+ int i, n = 0;
+ for (i=0; i<tok->nvalues; i++)
+ {
+ if ( bcf_double_is_missing_or_vector_end(tok->values[i]) ) continue;
+ if ( n < i ) tok->values[n] = tok->values[i];
+ n++;
+ }
+ if ( !n ) return 1;
+ if ( n==1 ) rtok->values[0] = tok->values[0];
+ else
+ {
+ qsort(tok->values, n, sizeof(double), compare_doubles);
+ rtok->values[0] = n % 2 ? tok->values[n/2] : (tok->values[n/2-1] + tok->values[n/2]) * 0.5;
+ }
+ rtok->nvalues = 1;
+ return 1;
+}
+static int func_stddev(filter_t *flt, bcf1_t *line, token_t *rtok, token_t **stack, int nstack)
+{
+ token_t *tok = stack[nstack - 1];
+ rtok->nvalues = 0;
+ if ( !tok->nvalues ) return 1;
+ int i, n = 0;
+ for (i=0; i<tok->nvalues; i++)
+ {
+ if ( bcf_double_is_missing_or_vector_end(tok->values[i]) ) continue;
+ if ( n < i ) tok->values[n] = tok->values[i];
+ n++;
+ }
+ if ( !n ) return 1;
+ if ( n==1 ) rtok->values[0] = 0;
+ else
+ {
+ double sdev = 0, avg = 0;
+ for (i=0; i<n; i++) avg += tok->values[n];
+ avg /= n;
+ for (i=0; i<n; i++) sdev += (tok->values[n] - avg) * (tok->values[n] - avg);
+ rtok->values[0] = sqrt(sdev/n);
+ }
+ rtok->nvalues = 1;
+ return 1;
+}
static int func_sum(filter_t *flt, bcf1_t *line, token_t *rtok, token_t **stack, int nstack)
{
rtok->nvalues = 0;
double val = 0;
int i, n = 0;
for (i=0; i<tok->nvalues; i++)
- if ( !bcf_double_is_missing(tok->values[i]) ) { val += tok->values[i]; n++; }
+ if ( !bcf_double_is_missing_or_vector_end(tok->values[i]) ) { val += tok->values[i]; n++; }
if ( n )
{
rtok->values[0] = val;
int i;
for (i=0; i<tok->nvalues; i++)
if ( bcf_double_is_missing(tok->values[i]) ) bcf_double_set_missing(rtok->values[i]);
- else rtok->values[i] = fabs(tok->values[i]);
+ else if ( !bcf_double_is_vector_end(tok->values[i]) ) rtok->values[i] = fabs(tok->values[i]);
return 1;
}
static int func_count(filter_t *flt, bcf1_t *line, token_t *rtok, token_t **stack, int nstack)
{
token_t *tok = stack[nstack - 1];
- if ( !tok->nsamples ) error("COUNT() can be applied only on FORMAT fields\n");
-
int i, cnt = 0;
- for (i=0; i<tok->nsamples; i++)
- if ( tok->pass_samples[i] ) cnt++;
+ if ( !tok->nsamples )
+ {
+ if ( tok->is_str )
+ {
+ if ( tok->str_value.l ) cnt = 1;
+ for (i=0; i<tok->str_value.l; i++) if ( tok->str_value.s[i]==',' ) cnt++;
+ }
+ else
+ cnt = tok->nvalues;
+ }
+ else
+ {
+ for (i=0; i<tok->nsamples; i++)
+ if ( tok->pass_samples[i] ) cnt++;
+ }
rtok->nvalues = 1;
rtok->values[0] = cnt;
}
int idx1 = bcf_gt_allele(ptr[0]);
int idx2 = bcf_gt_allele(ptr[1]);
- if ( idx1>=line->n_allele ) error("Incorrect allele index at %s:%d, sample %s\n", bcf_seqname(flt->hdr,line),line->pos+1,flt->hdr->samples[i]);
- if ( idx2>=line->n_allele ) error("Incorrect allele index at %s:%d, sample %s\n", bcf_seqname(flt->hdr,line),line->pos+1,flt->hdr->samples[i]);
+ if ( idx1>=line->n_allele ) error("Incorrect allele index at %s:%"PRId64", sample %s\n", bcf_seqname(flt->hdr,line),(int64_t) line->pos+1,flt->hdr->samples[i]);
+ if ( idx2>=line->n_allele ) error("Incorrect allele index at %s:%"PRId64", sample %s\n", bcf_seqname(flt->hdr,line),(int64_t) line->pos+1,flt->hdr->samples[i]);
double *vals = tok->values + tok->nval1*i;
- if ( bcf_double_is_missing(vals[idx1]) || bcf_double_is_missing(vals[idx2]) )
+ if ( bcf_double_is_missing_or_vector_end(vals[idx1]) || bcf_double_is_missing_or_vector_end(vals[idx2]) )
{
bcf_double_set_missing(rtok->values[i]);
continue;
// the fields given explicitly: binom(AD[:0],AD[:1])
token_t *tok2 = stack[istack+1];
if ( tok->nval1!=1 || tok2->nval1!=1 )
- error("Expected one value per binom() argument, found %d and %d at %s:%d\n",tok->nval1,tok2->nval1, bcf_seqname(flt->hdr,line),line->pos+1);
+ error("Expected one value per binom() argument, found %d and %d at %s:%"PRId64"\n",tok->nval1,tok2->nval1, bcf_seqname(flt->hdr,line),(int64_t) line->pos+1);
for (i=0; i<rtok->nsamples; i++)
{
if ( !rtok->usmpl[i] ) continue;
double *ptr1 = tok->values + tok->nval1*i;
double *ptr2 = tok2->values + tok2->nval1*i;
- if ( bcf_double_is_missing(ptr1[0]) || bcf_double_is_missing(ptr2[0]) )
+ if ( bcf_double_is_missing_or_vector_end(ptr1[0]) || bcf_double_is_missing_or_vector_end(ptr2[0]) )
{
bcf_double_set_missing(rtok->values[i]);
continue;
ptr2 = &tok2->values[0];
}
}
- if ( !ptr1 || !ptr2 || bcf_double_is_missing(ptr1[0]) || bcf_double_is_missing(ptr2[0]) )
+ if ( !ptr1 || !ptr2 || bcf_double_is_missing_or_vector_end(ptr1[0]) || bcf_double_is_missing_or_vector_end(ptr2[0]) )
bcf_double_set_missing(rtok->values[0]);
else
{
}
return rtok->nargs;
}
+static int func_phred(filter_t *flt, bcf1_t *line, token_t *rtok, token_t **stack, int nstack)
+{
+ token_t *tok = stack[nstack - 1];
+ if ( tok->is_str ) error("PHRED() can be applied only on numeric values\n");
+
+ rtok->nsamples = tok->nsamples;
+ rtok->nval1 = tok->nval1;
+ memcpy(rtok->pass_samples, tok->pass_samples, rtok->nsamples*sizeof(*rtok->pass_samples));
+ assert(tok->usmpl);
+ if ( !rtok->usmpl )
+ {
+ rtok->usmpl = (uint8_t*) malloc(tok->nsamples*sizeof(*rtok->usmpl));
+ memcpy(rtok->usmpl, tok->usmpl, tok->nsamples*sizeof(*rtok->usmpl));
+ }
+ rtok->nvalues = tok->nvalues;
+ if ( !tok->nvalues ) return 1;
+
+ hts_expand(double, rtok->nvalues, rtok->mvalues, rtok->values);
+ int i;
+ for (i=0; i<tok->nvalues; i++)
+ if ( bcf_double_is_missing_or_vector_end(tok->values[i]) ) bcf_double_set_missing(rtok->values[i]);
+ else rtok->values[i] = -4.34294481903*log(tok->values[i]);
+
+ return 1;
+}
inline static void tok_init_values(token_t *atok, token_t *btok, token_t *rtok)
{
token_t *tok = atok->nvalues > btok->nvalues ? atok : btok;
assert( atok->nsamples==btok->nsamples ); \
for (i=0; i<atok->nvalues; i++) \
{ \
- if ( bcf_double_is_missing(atok->values[i]) || bcf_double_is_missing(btok->values[i]) ) \
+ if ( bcf_double_is_missing_or_vector_end(atok->values[i]) || bcf_double_is_missing_or_vector_end(btok->values[i]) ) \
{ \
bcf_double_set_missing(rtok->values[i]); \
continue; \
token_t *xtok = atok->nsamples ? atok : btok; \
token_t *ytok = atok->nsamples ? btok : atok; \
assert( ytok->nvalues==1 ); \
- if ( !bcf_double_is_missing(ytok->values[0]) ) \
+ if ( !bcf_double_is_missing_or_vector_end(ytok->values[0]) ) \
{ \
for (i=0; i<xtok->nvalues; i++) \
{ \
- if ( bcf_double_is_missing(xtok->values[i]) ) \
+ if ( bcf_double_is_missing_or_vector_end(xtok->values[i]) ) \
{ \
bcf_double_set_missing(rtok->values[i]); \
continue; \
{ \
token_t *rtok = _rtok; \
int i, j, k; \
- assert( !atok->nsamples || !btok->nsamples ); \
tok_init_samples(atok, btok, rtok); \
if ( !atok->nsamples && !btok->nsamples ) \
{ \
token_t *tok = atok->nvalues ? atok : btok; \
for (j=0; j<tok->nvalues; j++) \
{ \
- if ( bcf_double_is_missing(tok->values[j]) ) \
+ if ( bcf_double_is_missing_or_vector_end(tok->values[j]) ) \
{ \
if ( missing_logic[2] ) { rtok->pass_site = 1; break; } \
} \
{ \
for (i=0; i<atok->nvalues; i++) \
{ \
- int amiss = bcf_double_is_missing(atok->values[i]) ? 1 : 0; \
+ int amiss = bcf_double_is_missing_or_vector_end(atok->values[i]) ? 1 : 0; \
for (j=0; j<btok->nvalues; j++) \
{ \
- int nmiss = amiss + (bcf_double_is_missing(btok->values[j]) ? 1 : 0); \
+ int nmiss = amiss + (bcf_double_is_missing_or_vector_end(btok->values[j]) ? 1 : 0); \
if ( nmiss ) \
{ \
if ( missing_logic[nmiss] ) { rtok->pass_site = 1; i = atok->nvalues; break; } \
} \
- else if ( atok->values[i] CMP_OP btok->values[j] ) { rtok->pass_site = 1; i = atok->nvalues; break; } \
+ else if ( atok->values[i] > 16777216 || btok->values[j] > 16777216 ) /* Ugly, see #871 */ \
+ { \
+ if ( atok->values[i] CMP_OP btok->values[j] ) { rtok->pass_site = 1; i = atok->nvalues; break; } \
+ } \
+ else if ( (float)atok->values[i] CMP_OP (float)btok->values[j] ) { rtok->pass_site = 1; i = atok->nvalues; break; } \
} \
} \
} \
{ \
int miss = 0; \
for (j=0; j<tok->nvalues; j++) \
- miss |= bcf_double_is_missing(tok->values[j]) ? 1 : 0; \
+ miss |= bcf_double_is_missing_or_vector_end(tok->values[j]) ? 1 : 0; \
if ( missing_logic[++miss] ) \
{ \
for (i=0; i<rtok->nsamples; i++) \
double *ptr = tok->values + i*tok->nval1; \
int miss = 0; \
for (j=0; j<tok->nval1; j++) \
- miss |= bcf_double_is_missing(ptr[j]) ? 1 : 0; \
+ miss |= bcf_double_is_missing_or_vector_end(ptr[j]) ? 1 : 0; \
if ( missing_logic[++miss] ) { rtok->pass_samples[i] = missing_logic[miss]; rtok->pass_site = 1; } \
} \
} \
+ else if ( atok->nsamples && btok->nsamples ) \
+ { \
+ if ( atok->nval1!=btok->nval1 ) error("Incompatible number of per-sample values in comparison: %d vs %d\n",atok->nval1,btok->nval1); \
+ if ( atok->nsamples!=btok->nsamples ) error("Incompatible number samples in comparison: %d vs %d\n",atok->nsamples,btok->nsamples); \
+ for (i=0; i<atok->nsamples; i++) \
+ { \
+ if ( !atok->usmpl[i] || !btok->usmpl[i] ) { rtok->usmpl[i] = 0; continue; } \
+ double *aptr = atok->values + i*atok->nval1; \
+ double *bptr = btok->values + i*btok->nval1; \
+ for (j=0; j<atok->nval1; j++) \
+ { \
+ int nmiss = bcf_double_is_missing_or_vector_end(aptr[j]) ? 1 : 0; \
+ if ( nmiss && !missing_logic[0] ) continue; /* any is missing => result is false */ \
+ nmiss += (bcf_double_is_missing_or_vector_end(bptr[j]) ? 1 : 0); \
+ if ( nmiss ) \
+ { \
+ if ( missing_logic[nmiss] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; break; } \
+ } \
+ else if ( aptr[j] > 16777216 || bptr[j] > 16777216 ) /* Ugly, see #871 */ \
+ { \
+ if ( aptr[j] CMP_OP bptr[j] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; break; } \
+ } \
+ else if ( (float)aptr[j] CMP_OP (float)bptr[j] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; break; } \
+ } \
+ } \
+ } \
else \
{ \
token_t *xtok = atok->nsamples ? atok : btok; \
double *yptr = ytok->values + i*ytok->nval1; \
for (j=0; j<xtok->nval1; j++) \
{ \
- int miss = bcf_double_is_missing(xptr[j]) ? 1 : 0; \
+ int miss = bcf_double_is_missing_or_vector_end(xptr[j]) ? 1 : 0; \
if ( miss && !missing_logic[0] ) continue; /* any is missing => result is false */ \
for (k=0; k<ytok->nvalues; k++) \
{ \
- int nmiss = miss + (bcf_double_is_missing(yptr[k]) ? 1 : 0); \
+ int nmiss = miss + (bcf_double_is_missing_or_vector_end(yptr[k]) ? 1 : 0); \
if ( nmiss ) \
{ \
if ( missing_logic[nmiss] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; j = xtok->nval1; break; } \
} \
- else if ( xptr[j] CMP_OP yptr[k] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; j = xtok->nval1; break; } \
+ else if ( xptr[j] > 16777216 || yptr[k] > 16777216 ) /* Ugly, see #871 */ \
+ { \
+ if ( xptr[j] CMP_OP yptr[k] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; j = xtok->nval1; break; } \
+ } \
+ else if ( (float)xptr[j] CMP_OP (float)yptr[k] ) { rtok->pass_samples[i] = 1; rtok->pass_site = 1; j = xtok->nval1; break; } \
} \
} \
} \
int *idxs2 = NULL, nidxs2 = 0, idx2 = 0;
int set_samples = 0;
- char *colon = rindex(tag_idx, ':');
+ char *colon = strrchr(tag_idx, ':');
if ( tag_idx[0]=='@' ) // file list with sample names
{
if ( !is_fmt ) error("Could not parse \"%s\". (Not a FORMAT tag yet a sample list provided.)\n", ori);
char *fname = expand_path(tag_idx+1);
+#ifdef _WIN32
+ if (fname && strlen(fname) > 2 && fname[1] == ':') // Deal with Windows paths, such as 'C:\..'
+ colon = strrchr(fname+2, ':');
+#endif
int nsmpl;
char **list = hts_readlist(fname, 1, &nsmpl);
if ( !list && colon )
tok->idxs = idxs2;
tok->nidxs = nidxs2;
tok->idx = idx2;
- colon = rindex(fname, ':');
+ colon = strrchr(fname, ':');
*colon = 0;
list = hts_readlist(fname, 1, &nsmpl);
}
}
static int filters_init1(filter_t *filter, char *str, int len, token_t *tok)
{
+ tok->tag_type = -1;
tok->tok_type = TOK_VAL;
tok->hdr_id = -1;
tok->pass_site = -1;
tok->comparator = filters_cmp_filter;
tok->tag = strdup("FILTER");
filter->max_unpack |= BCF_UN_FLT;
+ tok->tag_type = BCF_HL_FLT;
return 0;
}
else if ( !strncasecmp(str,"ID",len) || !strncasecmp(str,"%ID",len) /* for backward compatibility */ )
tok->tag = strdup("ID");
return 0;
}
+ else if ( !strncasecmp(str,"CHROM",len) )
+ {
+ tok->setter = &filters_set_chrom;
+ tok->tag = strdup("CHROM");
+ return 0;
+ }
else if ( !strncasecmp(str,"POS",len) )
{
tok->setter = &filters_set_pos;
}
else if ( !strncasecmp(str,"N_MISSING",len) )
{
+ filter->max_unpack |= BCF_UN_FMT;
tok->setter = &filters_set_nmissing;
tok->tag = strdup("N_MISSING");
return 0;
}
else if ( !strncasecmp(str,"F_MISSING",len) )
{
+ filter->max_unpack |= BCF_UN_FMT;
tok->setter = &filters_set_nmissing;
tok->tag = strdup("F_MISSING");
return 0;
for (i=0; i<tok->nsamples; i++) tok->usmpl[i] = 1;
}
- tok->type = is_fmt ? BCF_HL_FMT : BCF_HL_INFO;
+ tok->tag_type = is_fmt ? BCF_HL_FMT : BCF_HL_INFO;
if ( is_fmt ) filter->max_unpack |= BCF_UN_FMT;
if ( tok->hdr_id>=0 )
{
free(tmp.s);
return 0;
}
+ else if ( !strcasecmp(tmp.s,"ILEN") || !strcasecmp(tmp.s,"%ILEN") )
+ {
+ filter->max_unpack |= BCF_UN_STR;
+ tok->setter = &filters_set_ilen;
+ tok->tag = strdup("ILEN");
+ free(tmp.s);
+ return 0;
+ }
// is it a value? Here we parse as integer/float separately and use strtof
// rather than strtod, because the more accurate double representation
// would invalidate floating point comparisons like QUAL=59.2, obtained via
- // htslib/vcf parser
+ // htslib/vcf parser.
+ // Update: use strtod() and force floats only in comparisons
char *end;
tok->threshold = strtol(tmp.s, &end, 10); // integer?
if ( end - tmp.s != strlen(tmp.s) )
{
errno = 0;
- tok->threshold = strtof(tmp.s, &end); // float?
+ tok->threshold = strtod(tmp.s, &end); // float?
if ( errno!=0 || end!=tmp.s+len ) error("[%s:%d %s] Error: the tag \"%s\" is not defined in the VCF header\n", __FILE__,__LINE__,__FUNCTION__,tmp.s);
}
tok->is_constant = 1;
if ( ret==-1 ) error("Missing quotes in: %s\n", str);
// fprintf(bcftools_stderr,"token=[%c] .. [%s] %d\n", TOKEN_STRING[ret], tmp, len);
- // int i; for (i=0; i<nops; i++) fprintf(bcftools_stderr," .%c", TOKEN_STRING[ops[i]]); fprintf(bcftools_stderr,"\n");
+ // int i; for (i=0; i<nops; i++) fprintf(bcftools_stderr," .%c", TOKEN_STRING[ops[i].tok_type]); fprintf(bcftools_stderr,"\n");
if ( ret==TOK_LFT ) // left bracket
{
tok->hdr_id = -1;
tok->pass_site = -1;
tok->threshold = -1.0;
- if ( !strncasecmp(tmp-len,"N_PASS",6) ) { tok->func = func_npass; tok->tag = strdup("N_PASS"); }
- else if ( !strncasecmp(tmp-len,"F_PASS",6) ) { tok->func = func_npass; tok->tag = strdup("F_PASS"); }
+ if ( !strncasecmp(tmp-len,"N_PASS",6) )
+ {
+ filter->max_unpack |= BCF_UN_FMT;
+ tok->func = func_npass;
+ tok->tag = strdup("N_PASS");
+ }
+ else if ( !strncasecmp(tmp-len,"F_PASS",6) )
+ {
+ filter->max_unpack |= BCF_UN_FMT;
+ tok->func = func_npass;
+ tok->tag = strdup("F_PASS");
+ }
else error("The function \"%s\" is not supported\n", tmp-len);
continue;
}
// list of operators and convert the strings (e.g. "PASS") to BCF ids. The string value token must be
// just before or after the FILTER token and they must be followed with a comparison operator.
// At this point we also initialize regex expressions which, in RPN, must preceed the LIKE/NLIKE operator.
- // Additionally, treat "." as missing value rather than a string in numeric equalities.
+ // Additionally, treat "." as missing value rather than a string in numeric equalities; that
+ // @file is only used with ID; etc.
// This code is fragile: improve me.
int i;
for (i=0; i<nout; i++)
if ( i+1<nout && (out[i].tok_type==TOK_LT || out[i].tok_type==TOK_BT) && out[i+1].tok_type==TOK_EQ )
error("Error parsing the expression: \"%s\"\n", filter->str);
+ if ( out[i].hash )
+ {
+ int j = out[i+1].tok_type==TOK_VAL ? i+1 : i-1;
+ if ( out[j].comparator!=filters_cmp_id )
+ error("Error: could not parse the expression. Note that the \"@file_name\" syntax can be currently used with ID column only.\n");
+ }
if ( out[i].tok_type==TOK_OR || out[i].tok_type==TOK_OR_VEC )
out[i].func = vector_logic_or;
if ( out[i].tok_type==TOK_AND || out[i].tok_type==TOK_AND_VEC )
int set_missing = 0;
if ( out[k].hdr_id>0 )
{
- int type = bcf_hdr_id2type(filter->hdr,out[k].type,out[k].hdr_id);
+ int type = bcf_hdr_id2type(filter->hdr,out[k].tag_type,out[k].hdr_id);
if ( type==BCF_HT_INT ) set_missing = 1;
else if ( type==BCF_HT_REAL ) set_missing = 1;
}
}
if ( out[i].tok_type!=TOK_VAL ) continue;
if ( !out[i].tag ) continue;
- if ( !strcmp(out[i].tag,"TYPE") )
+ if ( out[i].setter==filters_set_type )
{
if ( i+1==nout ) error("Could not parse the expression: %s\n", filter->str);
int itok, ival;
else if ( !strcasecmp(out[ival].key,"mnp") || !strcasecmp(out[ival].key,"mnps") ) { out[ival].threshold = VCF_MNP<<1; out[ival].is_str = 0; }
else if ( !strcasecmp(out[ival].key,"other") ) { out[ival].threshold = VCF_OTHER<<1; out[ival].is_str = 0; }
else if ( !strcasecmp(out[ival].key,"bnd") ) { out[ival].threshold = VCF_BND<<1; out[ival].is_str = 0; }
+ else if ( !strcasecmp(out[ival].key,"overlap") ) { out[ival].threshold = VCF_OVERLAP<<1; out[ival].is_str = 0; }
else if ( !strcasecmp(out[ival].key,"ref") ) { out[ival].threshold = 1; out[ival].is_str = 0; }
else error("The type \"%s\" not recognised: %s\n", out[ival].key, filter->str);
if ( out[itok].tok_type==TOK_LIKE || out[itok].tok_type==TOK_NLIKE ) out[itok].comparator = filters_cmp_bit_and;
else if ( !strcasecmp(out[ival].key,"r") ) { out[i].setter = filters_set_genotype2; out[ival].key[0]='r'; out[ival].key[1]=0; } // r
continue;
}
- if ( !strcmp(out[i].tag,"FILTER") )
+ if ( out[i].tag_type==BCF_HL_FLT )
{
if ( i+1==nout ) error("Could not parse the expression: %s\n", filter->str);
int itok = i, ival;
filter->nsamples = filter->max_unpack&BCF_UN_FMT ? bcf_hdr_nsamples(filter->hdr) : 0;
for (i=0; i<nout; i++)
{
- if ( out[i].tok_type==TOK_MAX ) { out[i].func = func_max; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_MIN ) { out[i].func = func_min; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_AVG ) { out[i].func = func_avg; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_SUM ) { out[i].func = func_sum; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_ABS ) { out[i].func = func_abs; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_CNT ) { out[i].func = func_count; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
- else if ( out[i].tok_type==TOK_LEN ) { out[i].func = func_strlen; out[i].tok_type = TOK_FUNC; out[i].tok_type = 1; }
+ if ( out[i].tok_type==TOK_MAX ) { out[i].func = func_max; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_MIN ) { out[i].func = func_min; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_AVG ) { out[i].func = func_avg; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_MEDIAN ) { out[i].func = func_median; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_AVG ) { out[i].func = func_avg; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_STDEV ) { out[i].func = func_stddev; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_SUM ) { out[i].func = func_sum; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_ABS ) { out[i].func = func_abs; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_CNT ) { out[i].func = func_count; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_LEN ) { out[i].func = func_strlen; out[i].tok_type = TOK_FUNC; }
+ else if ( out[i].tok_type==TOK_PHRED ) { out[i].func = func_phred; out[i].tok_type = TOK_FUNC; }
else if ( out[i].tok_type==TOK_BINOM ) { out[i].func = func_binom; out[i].tok_type = TOK_FUNC; }
else if ( out[i].tok_type==TOK_PERLSUB ) { out[i].func = perl_exec; out[i].tok_type = TOK_FUNC; }
hts_expand0(double,1,out[i].mvalues,out[i].values);
}
else
{
+ if ( is_str==1 ) error("Error: cannot use arithmetic operators to compare strings and numbers\n");
+
// Determine what to do with one [1] or both [2] sides missing. The first field [0] gives [1]|[2]
int missing_logic[] = {0,0,0};
if ( filter->filters[i].tok_type == TOK_EQ ) { missing_logic[0] = missing_logic[2] = 1; }
CMP_VECTORS(filter->flt_stack[nstack-2],filter->flt_stack[nstack-1],&filter->filters[i],>=,missing_logic)
else
error("todo: %s:%d .. type=%d\n", __FILE__,__LINE__,filter->filters[i].tok_type);
-
}
filter->flt_stack[nstack-2] = &filter->filters[i];
nstack--;
if ( gvcf->npl>0 )
bcf_update_format_int32(hdr, gvcf->line, "PL", gvcf->pl, gvcf->npl);
bcf_update_format_int32(hdr, gvcf->line, "DP", gvcf->dp, nsmpl);
- bcf_write1(fh, hdr, gvcf->line);
+ if ( bcf_write1(fh, hdr, gvcf->line)!=0 ) error("[%s] Error: failed to write the record\n", __func__);
gvcf->prev_range = 0;
gvcf->rid = -1;
gvcf->npl = 0;
if ( gvcf->npl>0 )
bcf_update_format_int32(hdr, gvcf->line, "PL", gvcf->pl, gvcf->npl);
bcf_update_format_int32(hdr, gvcf->line, "DP", gvcf->dp, nsmpl);
- bcf_write1(fh, hdr, gvcf->line);
+ if ( bcf_write1(fh, hdr, gvcf->line)!=0 ) error("[%s] Error: failed to write the record\n", __func__);
gvcf->prev_range = 0;
gvcf->rid = -1;
gvcf->npl = 0;
--- /dev/null
+// VariantKey
+//
+// hex.h
+//
+// @category Libraries
+// @author Nicola Asuni <nicola.asuni@genomicsplc.com>
+// @copyright 2017-2018 GENOMICS plc
+// @license MIT (see LICENSE)
+// @link https://github.com/genomicsplc/variantkey
+//
+// LICENSE
+//
+// Copyright (c) 2017-2018 GENOMICS plc
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+/**
+ * @file hex.h
+ * @brief Utility functions to manipulate strings.
+ *
+ * Collection of utility functions to manipulate strings.
+ */
+
+#ifndef ASTRING_H
+#define ASTRING_H
+
+#include <inttypes.h>
+#include <stdio.h>
+
+/** @brief Returns uint64_t hexadecimal string (16 characters).
+ *
+ * @param n Number to parse
+ * @param str String buffer to be returned (it must be sized 17 bytes at least).
+ *
+ * @return Upon successful return, these function returns the number of characters processed
+ * (excluding the null byte used to end output to strings).
+ * If the buffer size is not sufficient, then the return value is the number of characters required for
+ * buffer string, including the terminating null byte.
+ */
+static inline size_t hex_uint64_t(uint64_t n, char *str)
+{
+ return sprintf(str, "%016" PRIx64, n);
+}
+
+/** @brief Parses a 16 chars hexadecimal string and returns the code.
+ *
+ * @param s Hexadecimal string to parse (it must contain 16 hexadecimal characters).
+ *
+ * @return uint64_t unsigned integer number.
+ */
+static inline uint64_t parse_hex_uint64_t(const char *s)
+{
+ uint64_t v = 0;
+ uint8_t b;
+ size_t i;
+ for (i = 0; i < 16; i++)
+ {
+ b = s[i];
+ if (b >= 'a')
+ {
+ b -= ('a' - 10); // a-f
+ }
+ else
+ {
+ if (b >= 'A')
+ {
+ b -= ('A' - 10); // A-F
+ }
+ else
+ {
+ b -= '0'; // 0-9
+ }
+ }
+ v = ((v << 4) | b);
+ }
+ return v;
+}
+
+#endif // ASTRING_H
--- /dev/null
+[Files in this distribution outwith the cram/ subdirectory are distributed
+according to the terms of the following MIT/Expat license.]
+
+The MIT/Expat License
+
+Copyright (C) 2012-2019 Genome Research Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+
+[Files within the cram/ subdirectory in this distribution are distributed
+according to the terms of the following Modified 3-Clause BSD license.]
+
+The Modified-BSD License
+
+Copyright (C) 2012-2019 Genome Research Ltd.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+3. Neither the names Genome Research Ltd and Wellcome Trust Sanger Institute
+ nor the names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR ITS CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+[The use of a range of years within a copyright notice in this distribution
+should be interpreted as being equivalent to a list of years including the
+first and last year specified and all consecutive years between them.
+
+For example, a copyright notice that reads "Copyright (C) 2005, 2007-2009,
+2011-2012" should be interpreted as being identical to a notice that reads
+"Copyright (C) 2005, 2007, 2008, 2009, 2011, 2012" and a copyright notice
+that reads "Copyright (C) 2005-2012" should be interpreted as being identical
+to a notice that reads "Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012".]
--- /dev/null
+HTSlib is an implementation of a unified C library for accessing common file
+formats, such as SAM, CRAM, VCF, and BCF, used for high-throughput sequencing
+data. It is the core library used by samtools and bcftools.
+
+See INSTALL for building and installation instructions.
+++ /dev/null
-[Files in this distribution outwith the cram/ subdirectory are distributed
-according to the terms of the following MIT/Expat license.]
-
-The MIT/Expat License
-
-Copyright (C) 2012-2018 Genome Research Ltd.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
-
-[Files within the cram/ subdirectory in this distribution are distributed
-according to the terms of the following Modified 3-Clause BSD license.]
-
-The Modified-BSD License
-
-Copyright (C) 2012-2018 Genome Research Ltd.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-3. Neither the names Genome Research Ltd and Wellcome Trust Sanger Institute
- nor the names of its contributors may be used to endorse or promote products
- derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR ITS CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-[The use of a range of years within a copyright notice in this distribution
-should be interpreted as being equivalent to a list of years including the
-first and last year specified and all consecutive years between them.
-
-For example, a copyright notice that reads "Copyright (C) 2005, 2007-2009,
-2011-2012" should be interpreted as being identical to a notice that reads
-"Copyright (C) 2005, 2007, 2008, 2009, 2011, 2012" and a copyright notice
-that reads "Copyright (C) 2005-2012" should be interpreted as being identical
-to a notice that reads "Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012".]
+++ /dev/null
-HTSlib is an implementation of a unified C library for accessing common file
-formats, such as SAM, CRAM, VCF, and BCF, used for high-throughput sequencing
-data. It is the core library used by samtools and bcftools.
-
-See INSTALL for building and installation instructions.
#if USE_GPL
int main_polysomy(int argc, char *argv[]);
#endif
+#ifdef ENABLE_BCF_PLUGINS
int main_plugin(int argc, char *argv[]);
+#endif
int main_consensus(int argc, char *argv[]);
int main_csq(int argc, char *argv[]);
int bam_mpileup(int argc, char *argv[]);
.alias = "norm",
.help = "left-align and normalize indels"
},
+#ifdef ENABLE_BCF_PLUGINS
{ .func = main_plugin,
.alias = "plugin",
-#ifdef ENABLE_BCF_PLUGINS
.help = "user-defined plugins"
-#else
- /* Do not advertise when plugins disabled. */
- .help = "-user-defined plugins"
-#endif
},
+#endif
{ .func = main_vcfquery,
.alias = "query",
.help = "transform VCF/BCF into user-defined formats"
fprintf(fp,"\n");
}
+// This is a tricky one, but on Windows the filename wildcard expansion is done by
+// the application and not by the shell, as traditionally it never had a "shell".
+// Even now, DOS and Powershell do not do this expansion (but bash does).
+//
+// This means that Mingw/Msys implements code before main() that takes e.g. "*" and
+// expands it up to a list of matching filenames. This in turn breaks things like
+// specifying "*" as a region (all the unmapped reads). We take a hard line here -
+// filename expansion is the task of the shell, not our application!
+#ifdef _WIN32
+int _CRT_glob = 0;
+#endif
+
int main(int argc, char *argv[])
{
if (argc < 2) { usage(stderr); return 1; }
if (strcmp(argv[1], "version") == 0 || strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-v") == 0) {
- printf("bcftools %s\nUsing htslib %s\nCopyright (C) 2018 Genome Research Ltd.\n", bcftools_version(), hts_version());
+ printf("bcftools %s\nUsing htslib %s\nCopyright (C) 2019 Genome Research Ltd.\n", bcftools_version(), hts_version());
#if USE_GPL
printf("License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>\n");
#else
#if USE_GPL
int main_polysomy(int argc, char *argv[]);
#endif
+#ifdef ENABLE_BCF_PLUGINS
int main_plugin(int argc, char *argv[]);
+#endif
int main_consensus(int argc, char *argv[]);
int main_csq(int argc, char *argv[]);
int bam_mpileup(int argc, char *argv[]);
.alias = "norm",
.help = "left-align and normalize indels"
},
+#ifdef ENABLE_BCF_PLUGINS
{ .func = main_plugin,
.alias = "plugin",
-#ifdef ENABLE_BCF_PLUGINS
.help = "user-defined plugins"
-#else
- /* Do not advertise when plugins disabled. */
- .help = "-user-defined plugins"
-#endif
},
+#endif
{ .func = main_vcfquery,
.alias = "query",
.help = "transform VCF/BCF into user-defined formats"
fprintf(fp,"\n");
}
+// This is a tricky one, but on Windows the filename wildcard expansion is done by
+// the application and not by the shell, as traditionally it never had a "shell".
+// Even now, DOS and Powershell do not do this expansion (but bash does).
+//
+// This means that Mingw/Msys implements code before main() that takes e.g. "*" and
+// expands it up to a list of matching filenames. This in turn breaks things like
+// specifying "*" as a region (all the unmapped reads). We take a hard line here -
+// filename expansion is the task of the shell, not our application!
+#ifdef _WIN32
+int _CRT_glob = 0;
+#endif
+
int bcftools_main(int argc, char *argv[])
{
if (argc < 2) { usage(bcftools_stderr); return 1; }
if (strcmp(argv[1], "version") == 0 || strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-v") == 0) {
- fprintf(bcftools_stdout, "bcftools %s\nUsing htslib %s\nCopyright (C) 2018 Genome Research Ltd.\n", bcftools_version(), hts_version());
+ fprintf(bcftools_stdout, "bcftools %s\nUsing htslib %s\nCopyright (C) 2019 Genome Research Ltd.\n", bcftools_version(), hts_version());
#if USE_GPL
fprintf(bcftools_stdout, "License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>\n");
#else
THE SOFTWARE. */
#include <math.h>
+#include <inttypes.h>
#include <htslib/kfunc.h>
+#include <htslib/khash_str2int.h>
#include "call.h"
// Using priors for GTs does not seem to be mathematically justified. Although
// genotypes is reported instead.
#define FLAT_PDG_FOR_MISSING 0
-// Estimate QS (combined quality and allele frequencies) from PLs
-#define QS_FROM_PDG 0
-
void qcall_init(call_t *call) { return; }
void qcall_destroy(call_t *call) { return; }
free(call->trio[j][i]);
}
+static void init_sample_groups(call_t *call)
+{
+ int i, nsmpl = bcf_hdr_nsamples(call->hdr);
+ if ( !call->sample_groups )
+ {
+ // standard pooled calling, all samples in the same group
+ grp_t *grps = &call->smpl_grp;
+ grps->ngrp = 1;
+ grps->grp = (grp1_t*)calloc(grps->ngrp, sizeof(grp1_t));
+ grps->smpl2grp = (int*)calloc(nsmpl,sizeof(int));
+ }
+ else if ( !strcmp("-",call->sample_groups) )
+ {
+ // single-sample calling, each sample creates its own group
+ grp_t *grps = &call->smpl_grp;
+ grps->ngrp = nsmpl;
+ grps->grp = (grp1_t*)calloc(grps->ngrp, sizeof(grp1_t));
+ grps->smpl2grp = (int*)malloc(nsmpl*sizeof(int));
+ for (i=0; i<nsmpl; i++) grps->smpl2grp[i] = i;
+ }
+ else
+ {
+ int nlines;
+ char **lines = hts_readlist(call->sample_groups, 1, &nlines);
+ if ( !lines ) error("Could not read the file: %s\n", call->sample_groups);
+
+ uint32_t *smpl2grp1 = (uint32_t*)calloc(nsmpl,sizeof(uint32_t));
+ void *grp2idx = khash_str2int_init();
+
+ grp_t *grps = &call->smpl_grp;
+ for (i=0; i<nlines; i++)
+ {
+ char *ptr = lines[i];
+ while ( *ptr && *ptr!='\t' ) ptr++;
+ if ( !*ptr ) error("Could not parse the line in %s, expected a sample name followed by tab and a population name: %s\n",call->sample_groups,lines[i]);
+ *ptr = 0;
+ int ismpl = bcf_hdr_id2int(call->hdr, BCF_DT_SAMPLE, lines[i]);
+ if ( ismpl<0 ) continue;
+ if ( smpl2grp1[ismpl] ) error("Error: the sample \"%s\" is listed twice in %s\n", lines[i],call->sample_groups);
+ if ( !khash_str2int_has_key(grp2idx,ptr+1) )
+ {
+ khash_str2int_inc(grp2idx, ptr+1);
+ grps->ngrp++;
+ }
+ int igrp;
+ if ( khash_str2int_get(grp2idx, ptr+1, &igrp)==0 )
+ smpl2grp1[ismpl] = igrp+1;
+ else
+ error("This should not happen, fixme: %s\n",ptr+1);
+ }
+ khash_str2int_destroy(grp2idx);
+
+ grps->grp = (grp1_t*)calloc(grps->ngrp, sizeof(grp1_t));
+ grps->smpl2grp = (int*)malloc(nsmpl*sizeof(int));
+ for (i=0; i<nsmpl; i++)
+ {
+ if ( !smpl2grp1[i] ) error("Error: The sample \"%s\" is not listed in %s\n",call->hdr->samples[i],call->sample_groups);
+ grps->smpl2grp[i] = smpl2grp1[i] - 1;
+ }
+ free(smpl2grp1);
+ for (i=0; i<nlines; i++) free(lines[i]);
+ free(lines);
+ }
+}
+static void destroy_sample_groups(call_t *call)
+{
+ int i;
+ grp_t *grps = &call->smpl_grp;
+ for (i=0; i<grps->ngrp; i++)
+ free(grps->grp[i].qsum);
+ free(grps->grp);
+ free(grps->smpl2grp);
+}
+
void mcall_init(call_t *call)
{
call_init_pl2p(call);
- call->nqsum = 5;
- call->qsum = (float*) malloc(sizeof(float)*call->nqsum); // will be expanded later if ncessary
call->nals_map = 5;
call->als_map = (int*) malloc(sizeof(int)*call->nals_map);
call->npl_map = 5*(5+1)/2; // will be expanded later if necessary
call->theta = log(call->theta);
}
- return;
+ init_sample_groups(call);
}
void mcall_destroy(call_t *call)
{
+ destroy_sample_groups(call);
if (call->vcmp) vcmp_destroy(call->vcmp);
free(call->itmp);
mcall_destroy_trios(call);
free(call->GPs);
+ free(call->ADs);
free(call->GLs);
free(call->GQs);
free(call->anno16);
free(call->PLs);
- free(call->qsum);
free(call->als_map);
free(call->pl_map);
free(call->gts); free(call->cgts); free(call->ugts);
free(call->pdg);
free(call->als);
free(call->ac);
+ free(call->qsum);
return;
}
}
}
-/*
- Allele frequency estimated as:
- #A = \sum_i (2*P_AA + P_AB)
- F_A = #A / ( #A + #B )
- where i runs across all samples
-*/
-void estimate_qsum(call_t *call, bcf1_t *rec)
-{
- double *pdg = call->pdg;
- int ngts = rec->n_allele*(rec->n_allele+1)/2;
- int i,nsmpl = bcf_hdr_nsamples(call->hdr);
-
- hts_expand(float,rec->n_allele,call->nqsum,call->qsum);
- for (i=0; i<rec->n_allele; i++) call->qsum[i] = 0;
-
- for (i=0; i<nsmpl; i++)
- {
- int a, b, k = 0;
- for (a=0; a<rec->n_allele; a++)
- {
- for (b=0; b<=a; b++)
- {
- call->qsum[a] += pdg[k];
- call->qsum[b] += pdg[k];
- k++;
- }
- }
- pdg += ngts;
- }
- float sum = 0;
- for (i=0; i<rec->n_allele; i++) sum += call->qsum[i];
- if ( sum ) for (i=0; i<rec->n_allele; i++) call->qsum[i] /= sum;
-}
-
// Create mapping between old and new (trimmed) alleles
void init_allele_trimming_maps(call_t *call, int als, int nals)
{
// at most tri-allelic sites are considered. Returns the number of alleles.
static int mcall_find_best_alleles(call_t *call, int nals, int *out_als)
{
+ int j;
int ia,ib,ic; // iterators over up to three alleles
int max_als=0; // most likely combination of alleles
double ref_lk = 0, max_lk = -HUGE_VAL; // likelihood of the reference and of most likely combination of alleles
UPDATE_MAX_LKs(1<<ia, ia>0 && lk_tot_set);
}
+ grp_t *grps = &call->smpl_grp;
+
// Two alleles
if ( nals>1 )
{
for (ia=0; ia<nals; ia++)
{
- if ( call->qsum[ia]==0 ) continue;
+ if ( grps->ngrp==1 && grps->grp[0].qsum[ia]==0 ) continue;
int iaa = (ia+1)*(ia+2)/2-1;
for (ib=0; ib<ia; ib++)
{
- if ( call->qsum[ib]==0 ) continue;
+ if ( grps->ngrp==1 && grps->grp[0].qsum[ib]==0 ) continue;
double lk_tot = 0;
int lk_tot_set = 0;
- double fa = call->qsum[ia]/(call->qsum[ia]+call->qsum[ib]);
- double fb = call->qsum[ib]/(call->qsum[ia]+call->qsum[ib]);
- double fa2 = fa*fa;
- double fb2 = fb*fb;
- double fab = 2*fa*fb;
+ int ia_cov = 0, ib_cov = 0;
+ for (j=0; j<grps->ngrp; j++)
+ {
+ grp1_t *grp = &grps->grp[j];
+ if ( grp->qsum[ia] ) ia_cov = 1;
+ if ( grp->qsum[ib] ) ib_cov = 1;
+ if ( !grp->qsum[ia] && !grp->qsum[ib] ) { grp->dp = 0; continue; }
+ grp->dp = 1;
+ grp->fa = grp->qsum[ia]/(grp->qsum[ia]+grp->qsum[ib]);
+ grp->fb = grp->qsum[ib]/(grp->qsum[ia]+grp->qsum[ib]);
+ grp->fa2 = grp->fa*grp->fa;
+ grp->fb2 = grp->fb*grp->fb;
+ grp->fab = 2*grp->fa*grp->fb;
+ }
+ if ( !ia_cov || !ib_cov ) continue;
int isample, ibb = (ib+1)*(ib+2)/2-1, iab = iaa - ia + ib;
double *pdg = call->pdg;
for (isample=0; isample<nsmpl; isample++)
{
+ grp1_t *grp = &grps->grp[grps->smpl2grp[isample]];
+ if ( !grp->dp ) continue;
double val = 0;
if ( !call->ploidy || call->ploidy[isample]==2 )
- val = fa2*pdg[iaa] + fb2*pdg[ibb] + fab*pdg[iab];
+ val = grp->fa2*pdg[iaa] + grp->fb2*pdg[ibb] + grp->fab*pdg[iab];
else if ( call->ploidy && call->ploidy[isample]==1 )
- val = fa*pdg[iaa] + fb*pdg[ibb];
+ val = grp->fa*pdg[iaa] + grp->fb*pdg[ibb];
if ( val ) { lk_tot += log(val); lk_tot_set = 1; }
pdg += ngts;
}
{
for (ia=0; ia<nals; ia++)
{
- if ( call->qsum[ia]==0 ) continue;
+ if ( grps->ngrp==1 && grps->grp[0].qsum[ia]==0 ) continue;
int iaa = (ia+1)*(ia+2)/2-1;
for (ib=0; ib<ia; ib++)
{
- if ( call->qsum[ib]==0 ) continue;
+ if ( grps->ngrp==1 && grps->grp[0].qsum[ib]==0 ) continue;
int ibb = (ib+1)*(ib+2)/2-1;
int iab = iaa - ia + ib;
for (ic=0; ic<ib; ic++)
{
- if ( call->qsum[ic]==0 ) continue;
+ if ( grps->ngrp==1 && grps->grp[0].qsum[ic]==0 ) continue;
double lk_tot = 0;
int lk_tot_set = 1;
- double fa = call->qsum[ia]/(call->qsum[ia]+call->qsum[ib]+call->qsum[ic]);
- double fb = call->qsum[ib]/(call->qsum[ia]+call->qsum[ib]+call->qsum[ic]);
- double fc = call->qsum[ic]/(call->qsum[ia]+call->qsum[ib]+call->qsum[ic]);
- double fa2 = fa*fa;
- double fb2 = fb*fb;
- double fc2 = fc*fc;
- double fab = 2*fa*fb, fac = 2*fa*fc, fbc = 2*fb*fc;
+ int ia_cov = 0, ib_cov = 0, ic_cov = 0;
+ for (j=0; j<grps->ngrp; j++)
+ {
+ grp1_t *grp = &grps->grp[j];
+ if ( grp->qsum[ia] ) ia_cov = 1;
+ if ( grp->qsum[ib] ) ib_cov = 1;
+ if ( grp->qsum[ic] ) ic_cov = 1;
+ if ( !grp->qsum[ia] && !grp->qsum[ib] && !grp->qsum[ic] ) { grp->dp = 0; continue; }
+ grp->dp = 1;
+ grp->fa = grp->qsum[ia]/(grp->qsum[ia]+grp->qsum[ib]+grp->qsum[ic]);
+ grp->fb = grp->qsum[ib]/(grp->qsum[ia]+grp->qsum[ib]+grp->qsum[ic]);
+ grp->fc = grp->qsum[ic]/(grp->qsum[ia]+grp->qsum[ib]+grp->qsum[ic]);
+ grp->fa2 = grp->fa*grp->fa;
+ grp->fb2 = grp->fb*grp->fb;
+ grp->fc2 = grp->fc*grp->fc;
+ grp->fab = 2*grp->fa*grp->fb, grp->fac = 2*grp->fa*grp->fc, grp->fbc = 2*grp->fb*grp->fc;
+ }
+ if ( !ia_cov || !ib_cov || !ic_cov ) continue;
int isample, icc = (ic+1)*(ic+2)/2-1;
int iac = iaa - ia + ic, ibc = ibb - ib + ic;
double *pdg = call->pdg;
for (isample=0; isample<nsmpl; isample++)
{
+ grp1_t *grp = &grps->grp[grps->smpl2grp[isample]];
+ if ( !grp->dp ) continue;
double val = 0;
if ( !call->ploidy || call->ploidy[isample]==2 )
- val = fa2*pdg[iaa] + fb2*pdg[ibb] + fc2*pdg[icc] + fab*pdg[iab] + fac*pdg[iac] + fbc*pdg[ibc];
+ val = grp->fa2*pdg[iaa] + grp->fb2*pdg[ibb] + grp->fc2*pdg[icc] + grp->fab*pdg[iab] + grp->fac*pdg[iac] + grp->fbc*pdg[ibc];
else if ( call->ploidy && call->ploidy[isample]==1 )
- val = fa*pdg[iaa] + fb*pdg[ibb] + fc*pdg[icc];
+ val = grp->fa*pdg[iaa] + grp->fb*pdg[ibb] + grp->fc*pdg[icc];
if ( val ) { lk_tot += log(val); lk_tot_set = 1; }
pdg += ngts;
}
gts[1] = ploidy==2 ? bcf_gt_unphased(0) : bcf_int32_vector_end;
// Non-zero depth, determine the most likely genotype
+ grp1_t *grp = &call->smpl_grp.grp[call->smpl_grp.smpl2grp[isample]];
double best_lk = 0;
for (ia=0; ia<nals; ia++)
{
if ( !(out_als & 1<<ia) ) continue; // ia-th allele not in the final selection, skip
int iaa = (ia+1)*(ia+2)/2-1; // PL index of the ia/ia genotype
- double lk = ploidy==2 ? pdg[iaa]*call->qsum[ia]*call->qsum[ia] : pdg[iaa]*call->qsum[ia];
+ double lk = ploidy==2 ? pdg[iaa]*grp->qsum[ia]*grp->qsum[ia] : pdg[iaa]*grp->qsum[ia];
#if USE_PRIOR_FOR_GTS
if ( ia!=0 ) lk *= prior;
#endif
{
if ( !(out_als & 1<<ib) ) continue;
int iab = iaa - ia + ib;
- double lk = 2*pdg[iab]*call->qsum[ia]*call->qsum[ib];
+ double lk = 2*pdg[iab]*grp->qsum[ia]*grp->qsum[ib];
#if USE_PRIOR_FOR_GTS
if ( ia!=0 ) lk *= prior;
if ( ib!=0 ) lk *= prior;
for (i=0; i<nout_gts; i++) gls[i] = -HUGE_VAL;
+ grp1_t *grp = &call->smpl_grp.grp[call->smpl_grp.smpl2grp[isample]];
double sum_lk = 0;
double best_lk = 0;
for (ia=0; ia<nals; ia++)
if ( !(out_als & 1<<ia) ) continue; // ia-th allele not in the final selection, skip
int iaa = bcf_alleles2gt(ia,ia); // PL index of the ia/ia genotype
int idx = bcf_alleles2gt(call->als_map[ia],call->als_map[ia]);
- double lk = ploidy==2 ? pdg[iaa]*call->qsum[ia]*call->qsum[ia] : pdg[iaa]*call->qsum[ia];
+ double lk = ploidy==2 ? pdg[iaa]*grp->qsum[ia]*grp->qsum[ia] : pdg[iaa]*grp->qsum[ia];
sum_lk += lk;
gls[idx] = lk;
if ( best_lk < lk )
if ( !(out_als & 1<<ib) ) continue;
int iab = bcf_alleles2gt(ia,ib);
int idx = bcf_alleles2gt(call->als_map[ia],call->als_map[ib]);
- double lk = 2*pdg[iab]*call->qsum[ia]*call->qsum[ib];
+ double lk = 2*pdg[iab]*grp->qsum[ia]*grp->qsum[ib];
sum_lk += lk;
gls[idx] = lk;
if ( best_lk < lk )
//
static int mcall_constrain_alleles(call_t *call, bcf1_t *rec, int *unseen)
{
- bcf_sr_regions_t *tgt = call->srs->targets;
- if ( tgt->nals>5 ) error("Maximum accepted number of alleles is 5, got %d\n", tgt->nals);
- hts_expand(char*,tgt->nals+1,call->nals,call->als);
+ assert( call->tgt_als->n );
+ if ( call->tgt_als->n>5 ) error("Maximum accepted number of alleles is 5, got %d\n", call->tgt_als->n);
+ hts_expand(char*,call->tgt_als->n+1,call->nals,call->als);
int has_new = 0;
int i, j, nals = 1;
for (i=1; i<call->nals_map; i++) call->als_map[i] = -1;
- if ( vcmp_set_ref(call->vcmp, rec->d.allele[0], tgt->als[0]) < 0 )
- error("The reference alleles are not compatible at %s:%d .. %s vs %s\n", call->hdr->id[BCF_DT_CTG][rec->rid].key,rec->pos+1,tgt->als[0],rec->d.allele[0]);
+ if ( vcmp_set_ref(call->vcmp, rec->d.allele[0], call->tgt_als->allele[0]) < 0 )
+ error("The reference alleles are not compatible at %s:%d .. %s vs %s\n", call->hdr->id[BCF_DT_CTG][rec->rid].key,rec->pos+1,call->tgt_als->allele[0],rec->d.allele[0]);
// create mapping from new to old alleles
- call->als[0] = tgt->als[0];
+ call->als[0] = call->tgt_als->allele[0];
call->als_map[0] = 0;
- for (i=1; i<tgt->nals; i++)
+ for (i=1; i<call->tgt_als->n; i++)
{
- call->als[nals] = tgt->als[i];
- j = vcmp_find_allele(call->vcmp, rec->d.allele+1, rec->n_allele - 1, tgt->als[i]);
+ call->als[nals] = call->tgt_als->allele[i];
+ j = vcmp_find_allele(call->vcmp, rec->d.allele+1, rec->n_allele - 1, call->tgt_als->allele[i]);
- if ( j+1==*unseen ) { fprintf(stderr,"fixme? Cannot constrain to %s\n",tgt->als[i]); return -1; }
+ if ( j+1==*unseen )
+ {
+ fprintf(stderr,"Fixme? Cannot constrain to %d-th allele (%s). VCF=",i,call->tgt_als->allele[i]);
+ int k;
+ for (k=0; k<rec->n_allele; k++) fprintf(stderr,"%s%s",k==0?"":",",rec->d.allele[k]);
+ fprintf(stderr,"\tTAB=");
+ for (k=0; k<call->tgt_als->n; k++) fprintf(stderr,"%s%s",k==0?"":",",call->tgt_als->allele[k]);
+ fprintf(stderr,"\n");
+ return -1;
+ }
if ( j>=0 )
{
bcf_update_format_int32(call->hdr, rec, "PL", call->itmp, npls_new*nsmpl);
// update QS
- float qsum[5];
- int nqs = bcf_get_info_float(call->hdr, rec, "QS", &call->qsum, &call->nqsum);
+ int nqs = bcf_get_info_float(call->hdr, rec, "QS", &call->smpl_grp.grp[0].qsum, &call->smpl_grp.grp[0].nqsum);
+ hts_expand(float,nals,call->nqsum,call->qsum);
for (i=0; i<nals; i++)
- qsum[i] = call->als_map[i]<nqs ? call->qsum[call->als_map[i]] : 0;
- bcf_update_info_float(call->hdr, rec, "QS", qsum, nals);
+ call->qsum[i] = call->als_map[i]<nqs ? call->smpl_grp.grp[0].qsum[call->als_map[i]] : 0;
+ bcf_update_info_float(call->hdr, rec, "QS", call->qsum, nals);
+
+ // update any Number=R tags
+ void *tmp_ori = call->itmp, *tmp_new = call->PLs; // reusing PLs storage which is not used at this point
+ int ntmp_ori = call->n_itmp, ntmp_new = call->mPLs;
+ for (i=0; i<rec->n_fmt; i++)
+ {
+ bcf_fmt_t *fmt = &rec->d.fmt[i];
+ int vlen = bcf_hdr_id2length(call->hdr,BCF_HL_FMT,fmt->id);
+ if ( vlen!=BCF_VL_R ) continue; // not a Number=R tag
+
+ // NB:works only for BCF_HT_INT and BCF_HT_REAL
+ int type = bcf_hdr_id2type(call->hdr,BCF_HL_FMT,fmt->id);
+ assert( type==BCF_HT_INT || type==BCF_HT_REAL );
+ assert( sizeof(float)==sizeof(int32_t) );
+
+ const char *key = bcf_hdr_int2id(call->hdr,BCF_DT_ID,fmt->id);
+ int nret = bcf_get_format_values(call->hdr, rec, key, &tmp_ori, &ntmp_ori, type);
+ if (nret<=0) continue;
+ int nsmpl = bcf_hdr_nsamples(call->hdr);
+ int size1 = sizeof(float);
+ hts_expand(float, nsmpl * nals, ntmp_new, tmp_new);
+ for (j=0; j<nsmpl; j++)
+ {
+ uint8_t *ptr_ori = (uint8_t *) tmp_ori + j*size1*fmt->n;
+ uint8_t *ptr_new = (uint8_t *) tmp_new + j*nals*size1;
+ for (k=0; k<nals; k++)
+ {
+ uint8_t *dst = ptr_new + size1*k;
+ uint8_t *src = ptr_ori + size1*call->als_map[k];
+ memcpy(dst,src,size1);
+ }
+ }
+ nret = bcf_update_format(call->hdr, rec, key, tmp_new, nsmpl*nals, type);
+ assert( nret==0 );
+ }
+ call->PLs = (int32_t*) tmp_new;
+ call->mPLs = ntmp_new;
+ call->itmp = (int32_t*) tmp_ori;
+ call->n_itmp = ntmp_ori;
+
if ( *unseen ) *unseen = nals-1;
return 0;
*/
int mcall(call_t *call, bcf1_t *rec)
{
- int i, unseen = call->unseen;
+ int i,j, unseen = call->unseen;
// Force alleles when calling genotypes given alleles was requested
if ( call->flag & CALL_CONSTR_ALLELES && mcall_constrain_alleles(call, rec, &unseen)!=0 ) return -2;
hts_expand(double, call->nPLs, call->npdg, call->pdg);
set_pdg(call->pl2p, call->PLs, call->pdg, nsmpl, ngts, unseen);
- #if QS_FROM_PDG
- estimate_qsum(call, rec);
- #else
- // Get sum of qualities, serves as an AF estimate, f_x = QS/N in Eq. 1 in call-m math notes.
- int nqs = bcf_get_info_float(call->hdr, rec, "QS", &call->qsum, &call->nqsum);
+ // Get sum of qualities, serves as an AF estimate, f_x = QS/N in Eq. 1 in call-m math notes.
+ if ( call->smpl_grp.ngrp == 1 )
+ {
+ int nqs = bcf_get_info_float(call->hdr, rec, "QS", &call->smpl_grp.grp[0].qsum, &call->smpl_grp.grp[0].nqsum);
if ( nqs<=0 ) error("The QS annotation not present at %s:%d\n", bcf_seqname(call->hdr,rec),rec->pos+1);
if ( nqs < nals )
{
// Some of the listed alleles do not have the corresponding QS field. This is
- // typically ref-only site with X in ALT.
+ // typically ref-only site with <*> in ALT.
+ hts_expand(float,nals,call->smpl_grp.grp[0].nqsum,call->smpl_grp.grp[0].qsum);
+ for (i=nqs; i<nals; i++) call->smpl_grp.grp[0].qsum[i] = 0;
+ }
+ }
+ else
+ {
+ for (j=0; j<call->smpl_grp.ngrp; j++)
+ {
+ hts_expand(float,nals,call->smpl_grp.grp[j].nqsum,call->smpl_grp.grp[j].qsum);
+ memset(call->smpl_grp.grp[j].qsum, 0, sizeof(float)*nals);
+ }
+
+ int nad = bcf_get_format_int32(call->hdr, rec, "AD", &call->ADs, &call->nADs);
+ if ( nad<1 ) error("Error: FORMAT/AD is required with the -G option, mpileup must be run with -a AD\n");
+ nad /= bcf_hdr_nsamples(call->hdr);
+ hts_expand(float,nals,call->nqsum,call->qsum);
+ float qsum = 0;
+ for (i=0; i<bcf_hdr_nsamples(call->hdr); i++)
+ {
+ int32_t *ptr = call->ADs + i*nad;
+ for (j=0; j<nad; j++)
+ {
+ if ( ptr[j]==bcf_int32_vector_end ) break;
+ if ( ptr[j]==bcf_int32_missing ) call->qsum[j] = 0;
+ else { call->qsum[j] = ptr[j]; qsum += ptr[j]; }
+ }
+ for (; j<nals; j++) call->qsum[j] = 0;
+ if ( qsum )
+ for (j=0; j<nals; j++) call->qsum[j] /= qsum;
- hts_expand(float,nals,call->nqsum,call->qsum);
- for (i=nqs; i<nals; i++) call->qsum[i] = 0;
+ grp1_t *grp = &call->smpl_grp.grp[call->smpl_grp.smpl2grp[i]];
+ for (j=0; j<nals; j++)
+ grp->qsum[j] += call->qsum[j];
}
+ }
- // If available, take into account reference panel AFs
- if ( call->prior_AN && bcf_get_info_int32(call->hdr, rec, call->prior_AN ,&call->ac, &call->nac)==1 )
+ // If available, take into account reference panel AFs
+ if ( call->prior_AN && bcf_get_info_int32(call->hdr, rec, call->prior_AN ,&call->ac, &call->nac)==1 )
+ {
+ int an = call->ac[0];
+ if ( bcf_get_info_int32(call->hdr, rec, call->prior_AC ,&call->ac, &call->nac)==nals-1 )
{
- int an = call->ac[0];
- if ( bcf_get_info_int32(call->hdr, rec, call->prior_AC ,&call->ac, &call->nac)==nals-1 )
+ int ac0 = an; // number of alleles in the reference population
+ for (i=0; i<nals-1; i++)
{
- int ac0 = an; // number of alleles in the reference population
- for (i=0; i<nals-1; i++)
- {
- if ( call->ac[i]==bcf_int32_vector_end ) break;
- if ( call->ac[i]==bcf_int32_missing ) continue;
- ac0 -= call->ac[i];
- call->qsum[i+1] += call->ac[i]*0.5;
- }
- if ( ac0<0 ) error("Incorrect %s,%s values at %s:%d\n", call->prior_AN,call->prior_AC,bcf_seqname(call->hdr,rec),rec->pos+1);
- call->qsum[0] += ac0*0.5;
- for (i=0; i<nals; i++) call->qsum[i] /= nsmpl + 0.5*an;
+ if ( call->ac[i]==bcf_int32_vector_end ) break;
+ if ( call->ac[i]==bcf_int32_missing ) continue;
+ ac0 -= call->ac[i];
+ for (j=0; j<call->smpl_grp.ngrp; j++)
+ call->smpl_grp.grp[j].qsum[i+1] += call->ac[i]*0.5;
+ }
+ if ( ac0<0 ) error("Incorrect %s,%s values at %s:%d\n", call->prior_AN,call->prior_AC,bcf_seqname(call->hdr,rec),rec->pos+1);
+ for (j=0; j<call->smpl_grp.ngrp; j++)
+ call->smpl_grp.grp[j].qsum[0] += ac0*0.5;
+ for (i=0; i<nals; i++)
+ {
+ for (j=0; j<call->smpl_grp.ngrp; j++)
+ call->smpl_grp.grp[j].qsum[i] /= nsmpl + 0.5*an;
}
}
+ }
+ for (j=0; j<call->smpl_grp.ngrp; j++)
+ {
float qsum_tot = 0;
- for (i=0; i<nals; i++) qsum_tot += call->qsum[i];
-
- // Is this still necessary??
- //
- // if (0&& !call->qsum[0] )
- // {
- // // As P(RR)!=0 even for QS(ref)=0, we set QS(ref) to a small value,
- // // an equivalent of a single reference read.
- // if ( bcf_get_info_int32(call->hdr, rec, "DP", &call->itmp, &call->n_itmp)!=1 )
- // error("Could not read DP at %s:%d\n", call->hdr->id[BCF_DT_CTG][rec->rid].key,rec->pos+1);
- // if ( call->itmp[0] )
- // {
- // call->qsum[0] = 1.0 / call->itmp[0] / nsmpl;
- // qsum_tot += call->qsum[0];
- // }
- // }
-
- if ( qsum_tot ) for (i=0; i<nals; i++) call->qsum[i] /= qsum_tot;
- #endif
+ for (i=0; i<nals; i++) qsum_tot += call->smpl_grp.grp[j].qsum[i];
+ if ( qsum_tot ) for (i=0; i<nals; i++) call->smpl_grp.grp[j].qsum[i] /= qsum_tot;
+ }
bcf_update_info_int32(call->hdr, rec, "QS", NULL, 0); // remove QS tag
int out_als, nout;
if ( nals > 8*sizeof(out_als) )
{
- fprintf(stderr,"Too many alleles at %s:%d, skipping.\n", bcf_seqname(call->hdr,rec),rec->pos+1);
+ fprintf(stderr,"Too many alleles at %s:%"PRId64", skipping.\n", bcf_seqname(call->hdr,rec),(int64_t) rec->pos+1);
return 0;
}
nout = mcall_find_best_alleles(call, nals, &out_als);
{
if ( nout>4 )
{
- fprintf(stderr,"Too many alleles at %s:%d, skipping.\n", bcf_seqname(call->hdr,rec),rec->pos+1);
+ fprintf(stderr,"Too many alleles at %s:%"PRId64", skipping.\n", bcf_seqname(call->hdr,rec),(int64_t) rec->pos+1);
return 0;
}
mcall_call_trio_genotypes(call, rec, nals,nout,out_als);
THE SOFTWARE. */
#include <math.h>
+#include <inttypes.h>
#include <htslib/kfunc.h>
+#include <htslib/khash_str2int.h>
#include "call.h"
// Using priors for GTs does not seem to be mathematically justified. Although
// genotypes is reported instead.
#define FLAT_PDG_FOR_MISSING 0
-// Estimate QS (combined quality and allele frequencies) from PLs
-#define QS_FROM_PDG 0
-
void qcall_init(call_t *call) { return; }
void qcall_destroy(call_t *call) { return; }
free(call->trio[j][i]);
}
+static void init_sample_groups(call_t *call)
+{
+ int i, nsmpl = bcf_hdr_nsamples(call->hdr);
+ if ( !call->sample_groups )
+ {
+ // standard pooled calling, all samples in the same group
+ grp_t *grps = &call->smpl_grp;
+ grps->ngrp = 1;
+ grps->grp = (grp1_t*)calloc(grps->ngrp, sizeof(grp1_t));
+ grps->smpl2grp = (int*)calloc(nsmpl,sizeof(int));
+ }
+ else if ( !strcmp("-",call->sample_groups) )
+ {
+ // single-sample calling, each sample creates its own group
+ grp_t *grps = &call->smpl_grp;
+ grps->ngrp = nsmpl;
+ grps->grp = (grp1_t*)calloc(grps->ngrp, sizeof(grp1_t));
+ grps->smpl2grp = (int*)malloc(nsmpl*sizeof(int));
+ for (i=0; i<nsmpl; i++) grps->smpl2grp[i] = i;
+ }
+ else
+ {
+ int nlines;
+ char **lines = hts_readlist(call->sample_groups, 1, &nlines);
+ if ( !lines ) error("Could not read the file: %s\n", call->sample_groups);
+
+ uint32_t *smpl2grp1 = (uint32_t*)calloc(nsmpl,sizeof(uint32_t));
+ void *grp2idx = khash_str2int_init();
+
+ grp_t *grps = &call->smpl_grp;
+ for (i=0; i<nlines; i++)
+ {
+ char *ptr = lines[i];
+ while ( *ptr && *ptr!='\t' ) ptr++;
+ if ( !*ptr ) error("Could not parse the line in %s, expected a sample name followed by tab and a population name: %s\n",call->sample_groups,lines[i]);
+ *ptr = 0;
+ int ismpl = bcf_hdr_id2int(call->hdr, BCF_DT_SAMPLE, lines[i]);
+ if ( ismpl<0 ) continue;
+ if ( smpl2grp1[ismpl] ) error("Error: the sample \"%s\" is listed twice in %s\n", lines[i],call->sample_groups);
+ if ( !khash_str2int_has_key(grp2idx,ptr+1) )
+ {
+ khash_str2int_inc(grp2idx, ptr+1);
+ grps->ngrp++;
+ }
+ int igrp;
+ if ( khash_str2int_get(grp2idx, ptr+1, &igrp)==0 )
+ smpl2grp1[ismpl] = igrp+1;
+ else
+ error("This should not happen, fixme: %s\n",ptr+1);
+ }
+ khash_str2int_destroy(grp2idx);
+
+ grps->grp = (grp1_t*)calloc(grps->ngrp, sizeof(grp1_t));
+ grps->smpl2grp = (int*)malloc(nsmpl*sizeof(int));
+ for (i=0; i<nsmpl; i++)
+ {
+ if ( !smpl2grp1[i] ) error("Error: The sample \"%s\" is not listed in %s\n",call->hdr->samples[i],call->sample_groups);
+ grps->smpl2grp[i] = smpl2grp1[i] - 1;
+ }
+ free(smpl2grp1);
+ for (i=0; i<nlines; i++) free(lines[i]);
+ free(lines);
+ }
+}
+static void destroy_sample_groups(call_t *call)
+{
+ int i;
+ grp_t *grps = &call->smpl_grp;
+ for (i=0; i<grps->ngrp; i++)
+ free(grps->grp[i].qsum);
+ free(grps->grp);
+ free(grps->smpl2grp);
+}
+
void mcall_init(call_t *call)
{
call_init_pl2p(call);
- call->nqsum = 5;
- call->qsum = (float*) malloc(sizeof(float)*call->nqsum); // will be expanded later if ncessary
call->nals_map = 5;
call->als_map = (int*) malloc(sizeof(int)*call->nals_map);
call->npl_map = 5*(5+1)/2; // will be expanded later if necessary
call->theta = log(call->theta);
}
- return;
+ init_sample_groups(call);
}
void mcall_destroy(call_t *call)
{
+ destroy_sample_groups(call);
if (call->vcmp) vcmp_destroy(call->vcmp);
free(call->itmp);
mcall_destroy_trios(call);
free(call->GPs);
+ free(call->ADs);
free(call->GLs);
free(call->GQs);
free(call->anno16);
free(call->PLs);
- free(call->qsum);
free(call->als_map);
free(call->pl_map);
free(call->gts); free(call->cgts); free(call->ugts);
free(call->pdg);
free(call->als);
free(call->ac);
+ free(call->qsum);
return;
}
}
}
-/*
- Allele frequency estimated as:
- #A = \sum_i (2*P_AA + P_AB)
- F_A = #A / ( #A + #B )
- where i runs across all samples
-*/
-void estimate_qsum(call_t *call, bcf1_t *rec)
-{
- double *pdg = call->pdg;
- int ngts = rec->n_allele*(rec->n_allele+1)/2;
- int i,nsmpl = bcf_hdr_nsamples(call->hdr);
-
- hts_expand(float,rec->n_allele,call->nqsum,call->qsum);
- for (i=0; i<rec->n_allele; i++) call->qsum[i] = 0;
-
- for (i=0; i<nsmpl; i++)
- {
- int a, b, k = 0;
- for (a=0; a<rec->n_allele; a++)
- {
- for (b=0; b<=a; b++)
- {
- call->qsum[a] += pdg[k];
- call->qsum[b] += pdg[k];
- k++;
- }
- }
- pdg += ngts;
- }
- float sum = 0;
- for (i=0; i<rec->n_allele; i++) sum += call->qsum[i];
- if ( sum ) for (i=0; i<rec->n_allele; i++) call->qsum[i] /= sum;
-}
-
// Create mapping between old and new (trimmed) alleles
void init_allele_trimming_maps(call_t *call, int als, int nals)
{
// at most tri-allelic sites are considered. Returns the number of alleles.
static int mcall_find_best_alleles(call_t *call, int nals, int *out_als)
{
+ int j;
int ia,ib,ic; // iterators over up to three alleles
int max_als=0; // most likely combination of alleles
double ref_lk = 0, max_lk = -HUGE_VAL; // likelihood of the reference and of most likely combination of alleles
UPDATE_MAX_LKs(1<<ia, ia>0 && lk_tot_set);
}
+ grp_t *grps = &call->smpl_grp;
+
// Two alleles
if ( nals>1 )
{
for (ia=0; ia<nals; ia++)
{
- if ( call->qsum[ia]==0 ) continue;
+ if ( grps->ngrp==1 && grps->grp[0].qsum[ia]==0 ) continue;
int iaa = (ia+1)*(ia+2)/2-1;
for (ib=0; ib<ia; ib++)
{
- if ( call->qsum[ib]==0 ) continue;
+ if ( grps->ngrp==1 && grps->grp[0].qsum[ib]==0 ) continue;
double lk_tot = 0;
int lk_tot_set = 0;
- double fa = call->qsum[ia]/(call->qsum[ia]+call->qsum[ib]);
- double fb = call->qsum[ib]/(call->qsum[ia]+call->qsum[ib]);
- double fa2 = fa*fa;
- double fb2 = fb*fb;
- double fab = 2*fa*fb;
+ int ia_cov = 0, ib_cov = 0;
+ for (j=0; j<grps->ngrp; j++)
+ {
+ grp1_t *grp = &grps->grp[j];
+ if ( grp->qsum[ia] ) ia_cov = 1;
+ if ( grp->qsum[ib] ) ib_cov = 1;
+ if ( !grp->qsum[ia] && !grp->qsum[ib] ) { grp->dp = 0; continue; }
+ grp->dp = 1;
+ grp->fa = grp->qsum[ia]/(grp->qsum[ia]+grp->qsum[ib]);
+ grp->fb = grp->qsum[ib]/(grp->qsum[ia]+grp->qsum[ib]);
+ grp->fa2 = grp->fa*grp->fa;
+ grp->fb2 = grp->fb*grp->fb;
+ grp->fab = 2*grp->fa*grp->fb;
+ }
+ if ( !ia_cov || !ib_cov ) continue;
int isample, ibb = (ib+1)*(ib+2)/2-1, iab = iaa - ia + ib;
double *pdg = call->pdg;
for (isample=0; isample<nsmpl; isample++)
{
+ grp1_t *grp = &grps->grp[grps->smpl2grp[isample]];
+ if ( !grp->dp ) continue;
double val = 0;
if ( !call->ploidy || call->ploidy[isample]==2 )
- val = fa2*pdg[iaa] + fb2*pdg[ibb] + fab*pdg[iab];
+ val = grp->fa2*pdg[iaa] + grp->fb2*pdg[ibb] + grp->fab*pdg[iab];
else if ( call->ploidy && call->ploidy[isample]==1 )
- val = fa*pdg[iaa] + fb*pdg[ibb];
+ val = grp->fa*pdg[iaa] + grp->fb*pdg[ibb];
if ( val ) { lk_tot += log(val); lk_tot_set = 1; }
pdg += ngts;
}
{
for (ia=0; ia<nals; ia++)
{
- if ( call->qsum[ia]==0 ) continue;
+ if ( grps->ngrp==1 && grps->grp[0].qsum[ia]==0 ) continue;
int iaa = (ia+1)*(ia+2)/2-1;
for (ib=0; ib<ia; ib++)
{
- if ( call->qsum[ib]==0 ) continue;
+ if ( grps->ngrp==1 && grps->grp[0].qsum[ib]==0 ) continue;
int ibb = (ib+1)*(ib+2)/2-1;
int iab = iaa - ia + ib;
for (ic=0; ic<ib; ic++)
{
- if ( call->qsum[ic]==0 ) continue;
+ if ( grps->ngrp==1 && grps->grp[0].qsum[ic]==0 ) continue;
double lk_tot = 0;
int lk_tot_set = 1;
- double fa = call->qsum[ia]/(call->qsum[ia]+call->qsum[ib]+call->qsum[ic]);
- double fb = call->qsum[ib]/(call->qsum[ia]+call->qsum[ib]+call->qsum[ic]);
- double fc = call->qsum[ic]/(call->qsum[ia]+call->qsum[ib]+call->qsum[ic]);
- double fa2 = fa*fa;
- double fb2 = fb*fb;
- double fc2 = fc*fc;
- double fab = 2*fa*fb, fac = 2*fa*fc, fbc = 2*fb*fc;
+ int ia_cov = 0, ib_cov = 0, ic_cov = 0;
+ for (j=0; j<grps->ngrp; j++)
+ {
+ grp1_t *grp = &grps->grp[j];
+ if ( grp->qsum[ia] ) ia_cov = 1;
+ if ( grp->qsum[ib] ) ib_cov = 1;
+ if ( grp->qsum[ic] ) ic_cov = 1;
+ if ( !grp->qsum[ia] && !grp->qsum[ib] && !grp->qsum[ic] ) { grp->dp = 0; continue; }
+ grp->dp = 1;
+ grp->fa = grp->qsum[ia]/(grp->qsum[ia]+grp->qsum[ib]+grp->qsum[ic]);
+ grp->fb = grp->qsum[ib]/(grp->qsum[ia]+grp->qsum[ib]+grp->qsum[ic]);
+ grp->fc = grp->qsum[ic]/(grp->qsum[ia]+grp->qsum[ib]+grp->qsum[ic]);
+ grp->fa2 = grp->fa*grp->fa;
+ grp->fb2 = grp->fb*grp->fb;
+ grp->fc2 = grp->fc*grp->fc;
+ grp->fab = 2*grp->fa*grp->fb, grp->fac = 2*grp->fa*grp->fc, grp->fbc = 2*grp->fb*grp->fc;
+ }
+ if ( !ia_cov || !ib_cov || !ic_cov ) continue;
int isample, icc = (ic+1)*(ic+2)/2-1;
int iac = iaa - ia + ic, ibc = ibb - ib + ic;
double *pdg = call->pdg;
for (isample=0; isample<nsmpl; isample++)
{
+ grp1_t *grp = &grps->grp[grps->smpl2grp[isample]];
+ if ( !grp->dp ) continue;
double val = 0;
if ( !call->ploidy || call->ploidy[isample]==2 )
- val = fa2*pdg[iaa] + fb2*pdg[ibb] + fc2*pdg[icc] + fab*pdg[iab] + fac*pdg[iac] + fbc*pdg[ibc];
+ val = grp->fa2*pdg[iaa] + grp->fb2*pdg[ibb] + grp->fc2*pdg[icc] + grp->fab*pdg[iab] + grp->fac*pdg[iac] + grp->fbc*pdg[ibc];
else if ( call->ploidy && call->ploidy[isample]==1 )
- val = fa*pdg[iaa] + fb*pdg[ibb] + fc*pdg[icc];
+ val = grp->fa*pdg[iaa] + grp->fb*pdg[ibb] + grp->fc*pdg[icc];
if ( val ) { lk_tot += log(val); lk_tot_set = 1; }
pdg += ngts;
}
gts[1] = ploidy==2 ? bcf_gt_unphased(0) : bcf_int32_vector_end;
// Non-zero depth, determine the most likely genotype
+ grp1_t *grp = &call->smpl_grp.grp[call->smpl_grp.smpl2grp[isample]];
double best_lk = 0;
for (ia=0; ia<nals; ia++)
{
if ( !(out_als & 1<<ia) ) continue; // ia-th allele not in the final selection, skip
int iaa = (ia+1)*(ia+2)/2-1; // PL index of the ia/ia genotype
- double lk = ploidy==2 ? pdg[iaa]*call->qsum[ia]*call->qsum[ia] : pdg[iaa]*call->qsum[ia];
+ double lk = ploidy==2 ? pdg[iaa]*grp->qsum[ia]*grp->qsum[ia] : pdg[iaa]*grp->qsum[ia];
#if USE_PRIOR_FOR_GTS
if ( ia!=0 ) lk *= prior;
#endif
{
if ( !(out_als & 1<<ib) ) continue;
int iab = iaa - ia + ib;
- double lk = 2*pdg[iab]*call->qsum[ia]*call->qsum[ib];
+ double lk = 2*pdg[iab]*grp->qsum[ia]*grp->qsum[ib];
#if USE_PRIOR_FOR_GTS
if ( ia!=0 ) lk *= prior;
if ( ib!=0 ) lk *= prior;
for (i=0; i<nout_gts; i++) gls[i] = -HUGE_VAL;
+ grp1_t *grp = &call->smpl_grp.grp[call->smpl_grp.smpl2grp[isample]];
double sum_lk = 0;
double best_lk = 0;
for (ia=0; ia<nals; ia++)
if ( !(out_als & 1<<ia) ) continue; // ia-th allele not in the final selection, skip
int iaa = bcf_alleles2gt(ia,ia); // PL index of the ia/ia genotype
int idx = bcf_alleles2gt(call->als_map[ia],call->als_map[ia]);
- double lk = ploidy==2 ? pdg[iaa]*call->qsum[ia]*call->qsum[ia] : pdg[iaa]*call->qsum[ia];
+ double lk = ploidy==2 ? pdg[iaa]*grp->qsum[ia]*grp->qsum[ia] : pdg[iaa]*grp->qsum[ia];
sum_lk += lk;
gls[idx] = lk;
if ( best_lk < lk )
if ( !(out_als & 1<<ib) ) continue;
int iab = bcf_alleles2gt(ia,ib);
int idx = bcf_alleles2gt(call->als_map[ia],call->als_map[ib]);
- double lk = 2*pdg[iab]*call->qsum[ia]*call->qsum[ib];
+ double lk = 2*pdg[iab]*grp->qsum[ia]*grp->qsum[ib];
sum_lk += lk;
gls[idx] = lk;
if ( best_lk < lk )
//
static int mcall_constrain_alleles(call_t *call, bcf1_t *rec, int *unseen)
{
- bcf_sr_regions_t *tgt = call->srs->targets;
- if ( tgt->nals>5 ) error("Maximum accepted number of alleles is 5, got %d\n", tgt->nals);
- hts_expand(char*,tgt->nals+1,call->nals,call->als);
+ assert( call->tgt_als->n );
+ if ( call->tgt_als->n>5 ) error("Maximum accepted number of alleles is 5, got %d\n", call->tgt_als->n);
+ hts_expand(char*,call->tgt_als->n+1,call->nals,call->als);
int has_new = 0;
int i, j, nals = 1;
for (i=1; i<call->nals_map; i++) call->als_map[i] = -1;
- if ( vcmp_set_ref(call->vcmp, rec->d.allele[0], tgt->als[0]) < 0 )
- error("The reference alleles are not compatible at %s:%d .. %s vs %s\n", call->hdr->id[BCF_DT_CTG][rec->rid].key,rec->pos+1,tgt->als[0],rec->d.allele[0]);
+ if ( vcmp_set_ref(call->vcmp, rec->d.allele[0], call->tgt_als->allele[0]) < 0 )
+ error("The reference alleles are not compatible at %s:%d .. %s vs %s\n", call->hdr->id[BCF_DT_CTG][rec->rid].key,rec->pos+1,call->tgt_als->allele[0],rec->d.allele[0]);
// create mapping from new to old alleles
- call->als[0] = tgt->als[0];
+ call->als[0] = call->tgt_als->allele[0];
call->als_map[0] = 0;
- for (i=1; i<tgt->nals; i++)
+ for (i=1; i<call->tgt_als->n; i++)
{
- call->als[nals] = tgt->als[i];
- j = vcmp_find_allele(call->vcmp, rec->d.allele+1, rec->n_allele - 1, tgt->als[i]);
+ call->als[nals] = call->tgt_als->allele[i];
+ j = vcmp_find_allele(call->vcmp, rec->d.allele+1, rec->n_allele - 1, call->tgt_als->allele[i]);
- if ( j+1==*unseen ) { fprintf(bcftools_stderr,"fixme? Cannot constrain to %s\n",tgt->als[i]); return -1; }
+ if ( j+1==*unseen )
+ {
+ fprintf(bcftools_stderr,"Fixme? Cannot constrain to %d-th allele (%s). VCF=",i,call->tgt_als->allele[i]);
+ int k;
+ for (k=0; k<rec->n_allele; k++) fprintf(bcftools_stderr,"%s%s",k==0?"":",",rec->d.allele[k]);
+ fprintf(bcftools_stderr,"\tTAB=");
+ for (k=0; k<call->tgt_als->n; k++) fprintf(bcftools_stderr,"%s%s",k==0?"":",",call->tgt_als->allele[k]);
+ fprintf(bcftools_stderr,"\n");
+ return -1;
+ }
if ( j>=0 )
{
bcf_update_format_int32(call->hdr, rec, "PL", call->itmp, npls_new*nsmpl);
// update QS
- float qsum[5];
- int nqs = bcf_get_info_float(call->hdr, rec, "QS", &call->qsum, &call->nqsum);
+ int nqs = bcf_get_info_float(call->hdr, rec, "QS", &call->smpl_grp.grp[0].qsum, &call->smpl_grp.grp[0].nqsum);
+ hts_expand(float,nals,call->nqsum,call->qsum);
for (i=0; i<nals; i++)
- qsum[i] = call->als_map[i]<nqs ? call->qsum[call->als_map[i]] : 0;
- bcf_update_info_float(call->hdr, rec, "QS", qsum, nals);
+ call->qsum[i] = call->als_map[i]<nqs ? call->smpl_grp.grp[0].qsum[call->als_map[i]] : 0;
+ bcf_update_info_float(call->hdr, rec, "QS", call->qsum, nals);
+
+ // update any Number=R tags
+ void *tmp_ori = call->itmp, *tmp_new = call->PLs; // reusing PLs storage which is not used at this point
+ int ntmp_ori = call->n_itmp, ntmp_new = call->mPLs;
+ for (i=0; i<rec->n_fmt; i++)
+ {
+ bcf_fmt_t *fmt = &rec->d.fmt[i];
+ int vlen = bcf_hdr_id2length(call->hdr,BCF_HL_FMT,fmt->id);
+ if ( vlen!=BCF_VL_R ) continue; // not a Number=R tag
+
+ // NB:works only for BCF_HT_INT and BCF_HT_REAL
+ int type = bcf_hdr_id2type(call->hdr,BCF_HL_FMT,fmt->id);
+ assert( type==BCF_HT_INT || type==BCF_HT_REAL );
+ assert( sizeof(float)==sizeof(int32_t) );
+
+ const char *key = bcf_hdr_int2id(call->hdr,BCF_DT_ID,fmt->id);
+ int nret = bcf_get_format_values(call->hdr, rec, key, &tmp_ori, &ntmp_ori, type);
+ if (nret<=0) continue;
+ int nsmpl = bcf_hdr_nsamples(call->hdr);
+ int size1 = sizeof(float);
+ hts_expand(float, nsmpl * nals, ntmp_new, tmp_new);
+ for (j=0; j<nsmpl; j++)
+ {
+ uint8_t *ptr_ori = (uint8_t *) tmp_ori + j*size1*fmt->n;
+ uint8_t *ptr_new = (uint8_t *) tmp_new + j*nals*size1;
+ for (k=0; k<nals; k++)
+ {
+ uint8_t *dst = ptr_new + size1*k;
+ uint8_t *src = ptr_ori + size1*call->als_map[k];
+ memcpy(dst,src,size1);
+ }
+ }
+ nret = bcf_update_format(call->hdr, rec, key, tmp_new, nsmpl*nals, type);
+ assert( nret==0 );
+ }
+ call->PLs = (int32_t*) tmp_new;
+ call->mPLs = ntmp_new;
+ call->itmp = (int32_t*) tmp_ori;
+ call->n_itmp = ntmp_ori;
+
if ( *unseen ) *unseen = nals-1;
return 0;
*/
int mcall(call_t *call, bcf1_t *rec)
{
- int i, unseen = call->unseen;
+ int i,j, unseen = call->unseen;
// Force alleles when calling genotypes given alleles was requested
if ( call->flag & CALL_CONSTR_ALLELES && mcall_constrain_alleles(call, rec, &unseen)!=0 ) return -2;
hts_expand(double, call->nPLs, call->npdg, call->pdg);
set_pdg(call->pl2p, call->PLs, call->pdg, nsmpl, ngts, unseen);
- #if QS_FROM_PDG
- estimate_qsum(call, rec);
- #else
- // Get sum of qualities, serves as an AF estimate, f_x = QS/N in Eq. 1 in call-m math notes.
- int nqs = bcf_get_info_float(call->hdr, rec, "QS", &call->qsum, &call->nqsum);
+ // Get sum of qualities, serves as an AF estimate, f_x = QS/N in Eq. 1 in call-m math notes.
+ if ( call->smpl_grp.ngrp == 1 )
+ {
+ int nqs = bcf_get_info_float(call->hdr, rec, "QS", &call->smpl_grp.grp[0].qsum, &call->smpl_grp.grp[0].nqsum);
if ( nqs<=0 ) error("The QS annotation not present at %s:%d\n", bcf_seqname(call->hdr,rec),rec->pos+1);
if ( nqs < nals )
{
// Some of the listed alleles do not have the corresponding QS field. This is
- // typically ref-only site with X in ALT.
+ // typically ref-only site with <*> in ALT.
+ hts_expand(float,nals,call->smpl_grp.grp[0].nqsum,call->smpl_grp.grp[0].qsum);
+ for (i=nqs; i<nals; i++) call->smpl_grp.grp[0].qsum[i] = 0;
+ }
+ }
+ else
+ {
+ for (j=0; j<call->smpl_grp.ngrp; j++)
+ {
+ hts_expand(float,nals,call->smpl_grp.grp[j].nqsum,call->smpl_grp.grp[j].qsum);
+ memset(call->smpl_grp.grp[j].qsum, 0, sizeof(float)*nals);
+ }
+
+ int nad = bcf_get_format_int32(call->hdr, rec, "AD", &call->ADs, &call->nADs);
+ if ( nad<1 ) error("Error: FORMAT/AD is required with the -G option, mpileup must be run with -a AD\n");
+ nad /= bcf_hdr_nsamples(call->hdr);
+ hts_expand(float,nals,call->nqsum,call->qsum);
+ float qsum = 0;
+ for (i=0; i<bcf_hdr_nsamples(call->hdr); i++)
+ {
+ int32_t *ptr = call->ADs + i*nad;
+ for (j=0; j<nad; j++)
+ {
+ if ( ptr[j]==bcf_int32_vector_end ) break;
+ if ( ptr[j]==bcf_int32_missing ) call->qsum[j] = 0;
+ else { call->qsum[j] = ptr[j]; qsum += ptr[j]; }
+ }
+ for (; j<nals; j++) call->qsum[j] = 0;
+ if ( qsum )
+ for (j=0; j<nals; j++) call->qsum[j] /= qsum;
- hts_expand(float,nals,call->nqsum,call->qsum);
- for (i=nqs; i<nals; i++) call->qsum[i] = 0;
+ grp1_t *grp = &call->smpl_grp.grp[call->smpl_grp.smpl2grp[i]];
+ for (j=0; j<nals; j++)
+ grp->qsum[j] += call->qsum[j];
}
+ }
- // If available, take into account reference panel AFs
- if ( call->prior_AN && bcf_get_info_int32(call->hdr, rec, call->prior_AN ,&call->ac, &call->nac)==1 )
+ // If available, take into account reference panel AFs
+ if ( call->prior_AN && bcf_get_info_int32(call->hdr, rec, call->prior_AN ,&call->ac, &call->nac)==1 )
+ {
+ int an = call->ac[0];
+ if ( bcf_get_info_int32(call->hdr, rec, call->prior_AC ,&call->ac, &call->nac)==nals-1 )
{
- int an = call->ac[0];
- if ( bcf_get_info_int32(call->hdr, rec, call->prior_AC ,&call->ac, &call->nac)==nals-1 )
+ int ac0 = an; // number of alleles in the reference population
+ for (i=0; i<nals-1; i++)
{
- int ac0 = an; // number of alleles in the reference population
- for (i=0; i<nals-1; i++)
- {
- if ( call->ac[i]==bcf_int32_vector_end ) break;
- if ( call->ac[i]==bcf_int32_missing ) continue;
- ac0 -= call->ac[i];
- call->qsum[i+1] += call->ac[i]*0.5;
- }
- if ( ac0<0 ) error("Incorrect %s,%s values at %s:%d\n", call->prior_AN,call->prior_AC,bcf_seqname(call->hdr,rec),rec->pos+1);
- call->qsum[0] += ac0*0.5;
- for (i=0; i<nals; i++) call->qsum[i] /= nsmpl + 0.5*an;
+ if ( call->ac[i]==bcf_int32_vector_end ) break;
+ if ( call->ac[i]==bcf_int32_missing ) continue;
+ ac0 -= call->ac[i];
+ for (j=0; j<call->smpl_grp.ngrp; j++)
+ call->smpl_grp.grp[j].qsum[i+1] += call->ac[i]*0.5;
+ }
+ if ( ac0<0 ) error("Incorrect %s,%s values at %s:%d\n", call->prior_AN,call->prior_AC,bcf_seqname(call->hdr,rec),rec->pos+1);
+ for (j=0; j<call->smpl_grp.ngrp; j++)
+ call->smpl_grp.grp[j].qsum[0] += ac0*0.5;
+ for (i=0; i<nals; i++)
+ {
+ for (j=0; j<call->smpl_grp.ngrp; j++)
+ call->smpl_grp.grp[j].qsum[i] /= nsmpl + 0.5*an;
}
}
+ }
+ for (j=0; j<call->smpl_grp.ngrp; j++)
+ {
float qsum_tot = 0;
- for (i=0; i<nals; i++) qsum_tot += call->qsum[i];
-
- // Is this still necessary??
- //
- // if (0&& !call->qsum[0] )
- // {
- // // As P(RR)!=0 even for QS(ref)=0, we set QS(ref) to a small value,
- // // an equivalent of a single reference read.
- // if ( bcf_get_info_int32(call->hdr, rec, "DP", &call->itmp, &call->n_itmp)!=1 )
- // error("Could not read DP at %s:%d\n", call->hdr->id[BCF_DT_CTG][rec->rid].key,rec->pos+1);
- // if ( call->itmp[0] )
- // {
- // call->qsum[0] = 1.0 / call->itmp[0] / nsmpl;
- // qsum_tot += call->qsum[0];
- // }
- // }
-
- if ( qsum_tot ) for (i=0; i<nals; i++) call->qsum[i] /= qsum_tot;
- #endif
+ for (i=0; i<nals; i++) qsum_tot += call->smpl_grp.grp[j].qsum[i];
+ if ( qsum_tot ) for (i=0; i<nals; i++) call->smpl_grp.grp[j].qsum[i] /= qsum_tot;
+ }
bcf_update_info_int32(call->hdr, rec, "QS", NULL, 0); // remove QS tag
int out_als, nout;
if ( nals > 8*sizeof(out_als) )
{
- fprintf(bcftools_stderr,"Too many alleles at %s:%d, skipping.\n", bcf_seqname(call->hdr,rec),rec->pos+1);
+ fprintf(bcftools_stderr,"Too many alleles at %s:%"PRId64", skipping.\n", bcf_seqname(call->hdr,rec),(int64_t) rec->pos+1);
return 0;
}
nout = mcall_find_best_alleles(call, nals, &out_als);
{
if ( nout>4 )
{
- fprintf(bcftools_stderr,"Too many alleles at %s:%d, skipping.\n", bcf_seqname(call->hdr,rec),rec->pos+1);
+ fprintf(bcftools_stderr,"Too many alleles at %s:%"PRId64", skipping.\n", bcf_seqname(call->hdr,rec),(int64_t) rec->pos+1);
return 0;
}
mcall_call_trio_genotypes(call, rec, nals,nout,out_als);
/* mpileup.c -- mpileup subcommand. Previously bam_plcmd.c from samtools
- Copyright (C) 2008-2017 Genome Research Ltd.
+ Copyright (C) 2008-2018 Genome Research Ltd.
Portions copyright (C) 2009-2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include <string.h>
#include <strings.h>
#include <limits.h>
+#include <inttypes.h>
#include <errno.h>
#include <sys/stat.h>
#include <getopt.h>
if (ma->conf->fai && b->core.tid >= 0) {
has_ref = mplp_get_ref(ma, b->core.tid, &ref, &ref_len);
if (has_ref && ref_len <= b->core.pos) { // exclude reads outside of the reference sequence
- fprintf(stderr,"[%s] Skipping because %d is outside of %d [ref:%d]\n",
- __func__, b->core.pos, ref_len, b->core.tid);
+ fprintf(stderr,"[%s] Skipping because %"PRId64" is outside of %d [ref:%d]\n",
+ __func__, (int64_t) b->core.pos, ref_len, b->core.tid);
continue;
}
} else {
// Called once per new bam added to the pileup.
// We cache sample information here so we don't have to keep recomputing this
-// on each and every pileup column.
+// on each and every pileup column. If FMT/SCR annotation is requested, a flag
+// is set to indicate the presence of a soft clip.
//
// Cd is an arbitrary block of data we can write into, which ends up in
-// the pileup structures. We stash the sample ID there.
-static int pileup_constructor(void *data, const bam1_t *b, bam_pileup_cd *cd) {
+// the pileup structures. We stash the sample ID there:
+// has_soft_clip .. cd->i & 1
+// sample_id .. cd->i >> 1
+static int pileup_constructor(void *data, const bam1_t *b, bam_pileup_cd *cd)
+{
mplp_aux_t *ma = (mplp_aux_t *)data;
- cd->i = bam_smpl_get_sample_id(ma->conf->bsmpl, ma->bam_id, (bam1_t *)b);
+ cd->i = bam_smpl_get_sample_id(ma->conf->bsmpl, ma->bam_id, (bam1_t *)b) << 1;
+ if ( ma->conf->fmt_flag & (B2B_INFO_SCR|B2B_FMT_SCR) )
+ {
+ int i;
+ for (i=0; i<b->core.n_cigar; i++)
+ {
+ int cig = bam_get_cigar(b)[i] & BAM_CIGAR_MASK;
+ if ( cig!=BAM_CSOFT_CLIP ) continue;
+ cd->i |= 1;
+ break;
+ }
+ }
return 0;
}
for (j = 0; j < n_plp[i]; ++j) // iterate over all reads available at this position
{
const bam_pileup1_t *p = plp[i] + j;
- int id = p->cd.i;
+ int id = PLP_SAMPLE_ID(p->cd.i);
if (m->n_plp[id] == m->m_plp[id])
{
m->m_plp[id] = m->m_plp[id]? m->m_plp[id]<<1 : 8;
{
if ( !conf->gvcf )
{
- if ( rec ) bcf_write1(fp, hdr, rec);
+ if ( rec && bcf_write1(fp, hdr, rec)!=0 ) error("[%s] Error: failed to write the record to %s\n", __func__,conf->output_fname?conf->output_fname:"standard output");
return;
}
if ( rec->d.allele[1][0]=='<' && rec->d.allele[1][1]=='*' && rec->d.allele[1][2]=='>' ) is_ref = 1;
}
rec = gvcf_write(conf->gvcf, fp, hdr, rec, is_ref);
- if ( rec ) bcf_write1(fp,hdr,rec);
+ if ( rec && bcf_write1(fp,hdr,rec)!=0 ) error("[%s] Error: failed to write the record to %s\n", __func__,conf->output_fname?conf->output_fname:"standard output");
}
static int mpileup_reg(mplp_conf_t *conf, uint32_t beg, uint32_t end)
while ( (ret=bam_mplp_auto(conf->iter, &tid, &pos, conf->n_plp, conf->plp)) > 0)
{
- if ( end && (pos<beg || pos>end) ) continue;
+ if ( pos<beg || pos>end ) continue;
if ( conf->bed && tid >= 0 )
{
int overlap = regidx_overlap(conf->bed, hdr->target_name[tid], pos, pos, NULL);
bcf_hdr_append(conf->bcf_hdr,"##ALT=<ID=*,Description=\"Represents allele(s) other than observed.\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=INDEL,Number=0,Type=Flag,Description=\"Indicates that the variant is an INDEL.\">");
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IDV,Number=1,Type=Integer,Description=\"Maximum number of reads supporting an indel\">");
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IMF,Number=1,Type=Float,Description=\"Maximum fraction of reads supporting an indel\">");
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IDV,Number=1,Type=Integer,Description=\"Maximum number of raw reads supporting an indel\">");
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IMF,Number=1,Type=Float,Description=\"Maximum fraction of raw reads supporting an indel\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=DP,Number=1,Type=Integer,Description=\"Raw read depth\">");
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=VDB,Number=1,Type=Float,Description=\"Variant Distance Bias for filtering splice-site artefacts in RNA-seq data (bigger is better)\",Version=\"3\">");
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=RPB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Read Position Bias (bigger is better)\">");
+ if ( conf->fmt_flag&B2B_INFO_VDB )
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=VDB,Number=1,Type=Float,Description=\"Variant Distance Bias for filtering splice-site artefacts in RNA-seq data (bigger is better)\",Version=\"3\">");
+ if ( conf->fmt_flag&B2B_INFO_RPB )
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=RPB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Read Position Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=BQB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Base Quality Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQSB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality vs Strand Bias (bigger is better)\">");
if ( conf->fmt_flag&B2B_FMT_SP )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=SP,Number=1,Type=Integer,Description=\"Phred-scaled strand bias P-value\">");
if ( conf->fmt_flag&B2B_FMT_AD )
- bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=AD,Number=R,Type=Integer,Description=\"Allelic depths\">");
+ bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=AD,Number=R,Type=Integer,Description=\"Allelic depths (high-quality bases)\">");
if ( conf->fmt_flag&B2B_FMT_ADF )
- bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADF,Number=R,Type=Integer,Description=\"Allelic depths on the forward strand\">");
+ bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADF,Number=R,Type=Integer,Description=\"Allelic depths on the forward strand (high-quality bases)\">");
if ( conf->fmt_flag&B2B_FMT_ADR )
- bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADR,Number=R,Type=Integer,Description=\"Allelic depths on the reverse strand\">");
+ bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADR,Number=R,Type=Integer,Description=\"Allelic depths on the reverse strand (high-quality bases)\">");
if ( conf->fmt_flag&B2B_INFO_AD )
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=AD,Number=R,Type=Integer,Description=\"Total allelic depths\">");
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=AD,Number=R,Type=Integer,Description=\"Total allelic depths (high-quality bases)\">");
if ( conf->fmt_flag&B2B_INFO_ADF )
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADF,Number=R,Type=Integer,Description=\"Total allelic depths on the forward strand\">");
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADF,Number=R,Type=Integer,Description=\"Total allelic depths on the forward strand (high-quality bases)\">");
+ if ( conf->fmt_flag&B2B_INFO_SCR )
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=SCR,Number=1,Type=Integer,Description=\"Number of soft-clipped reads (at high-quality bases)\">");
+ if ( conf->fmt_flag&B2B_FMT_SCR )
+ bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=SCR,Number=1,Type=Integer,Description=\"Per-sample number of soft-clipped reads (at high-quality bases)\">");
if ( conf->fmt_flag&B2B_INFO_ADR )
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADR,Number=R,Type=Integer,Description=\"Total allelic depths on the reverse strand\">");
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADR,Number=R,Type=Integer,Description=\"Total allelic depths on the reverse strand (high-quality bases)\">");
if ( conf->gvcf )
gvcf_update_header(conf->gvcf, conf->bcf_hdr);
const char **smpl = bam_smpl_get_samples(conf->bsmpl, &nsmpl);
for (i=0; i<nsmpl; i++)
bcf_hdr_add_sample(conf->bcf_hdr, smpl[i]);
- bcf_hdr_write(conf->bcf_fp, conf->bcf_hdr);
+ if ( bcf_hdr_write(conf->bcf_fp, conf->bcf_hdr)!=0 ) error("[%s] Error: failed to write the header to %s\n",__func__,conf->output_fname?conf->output_fname:"standard output");
conf->bca = bcf_call_init(-1., conf->min_baseQ);
conf->bcr = (bcf_callret1_t*) calloc(nsmpl, sizeof(bcf_callret1_t));
conf->bca->min_frac = conf->min_frac;
conf->bca->min_support = conf->min_support;
conf->bca->per_sample_flt = conf->flag & MPLP_PER_SAMPLE;
+ conf->bca->fmt_flag = conf->fmt_flag;
conf->bc.bcf_hdr = conf->bcf_hdr;
conf->bc.n = nsmpl;
conf->bcr[i].ADF = conf->bc.ADF + (i+1)*B2B_MAX_ALLELES;
}
}
+ if ( conf->fmt_flag&(B2B_INFO_SCR|B2B_FMT_SCR) )
+ conf->bc.SCR = (int32_t*) malloc((nsmpl+1)*sizeof(*conf->bc.SCR));
}
// init mpileup
conf->iter = bam_mplp_init(conf->nfiles, mplp_func, (void**)conf->mplp_data);
if ( conf->flag & MPLP_SMART_OVERLAPS ) bam_mplp_init_overlaps(conf->iter);
+ fprintf(stderr, "[%s] maximum number of reads per input file set to -d %d\n", __func__, conf->max_depth);
if ( (double)conf->max_depth * conf->nfiles > 1<<20)
fprintf(stderr, "Warning: Potential memory hog, up to %.0fM reads in the pileup!\n", (double)conf->max_depth*conf->nfiles);
if ( (double)conf->max_depth * conf->nfiles / nsmpl < 250 )
if ( ireg++ > 0 )
{
conf->buf.l = 0;
- ksprintf(&conf->buf,"%s:%u-%u",conf->reg_itr->seq,conf->reg_itr->beg,conf->reg_itr->end);
+ ksprintf(&conf->buf,"%s:%u-%u",conf->reg_itr->seq,conf->reg_itr->beg+1,conf->reg_itr->end+1);
for (i=0; i<conf->nfiles; i++)
{
while ( regitr_loop(conf->reg_itr) );
}
else
- mpileup_reg(conf,0,0);
+ mpileup_reg(conf,0,UINT32_MAX);
flush_bcf_records(conf, conf->bcf_fp, conf->bcf_hdr, NULL);
bcf_destroy1(conf->bcf_rec);
if (conf->bcf_fp)
{
- hts_close(conf->bcf_fp);
+ if ( hts_close(conf->bcf_fp)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,conf->output_fname);
bcf_hdr_destroy(conf->bcf_hdr);
bcf_call_destroy(conf->bca);
free(conf->bc.PL);
free(conf->bc.DP4);
free(conf->bc.ADR);
free(conf->bc.ADF);
+ free(conf->bc.SCR);
free(conf->bc.fmt_arr);
free(conf->bcr);
}
files = (char**) realloc(files,nfiles*sizeof(char*));
files[nfiles-1] = strdup(buf);
}
- fclose(fh);
+ if ( fclose(fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,file_list);
if ( !nfiles )
{
fprintf(stderr,"No files read from %s\n", file_list);
else if ( !strcasecmp(tags[i],"AD") || !strcasecmp(tags[i],"FORMAT/AD") || !strcasecmp(tags[i],"FMT/AD") ) flag |= B2B_FMT_AD;
else if ( !strcasecmp(tags[i],"ADF") || !strcasecmp(tags[i],"FORMAT/ADF") || !strcasecmp(tags[i],"FMT/ADF") ) flag |= B2B_FMT_ADF;
else if ( !strcasecmp(tags[i],"ADR") || !strcasecmp(tags[i],"FORMAT/ADR") || !strcasecmp(tags[i],"FMT/ADR") ) flag |= B2B_FMT_ADR;
+ else if ( !strcasecmp(tags[i],"SCR") || !strcasecmp(tags[i],"FORMAT/SCR") || !strcasecmp(tags[i],"FMT/SCR") ) flag |= B2B_FMT_SCR;
+ else if ( !strcasecmp(tags[i],"INFO/SCR") ) flag |= B2B_INFO_SCR;
else if ( !strcasecmp(tags[i],"INFO/AD") ) flag |= B2B_INFO_AD;
else if ( !strcasecmp(tags[i],"INFO/ADF") ) flag |= B2B_INFO_ADF;
else if ( !strcasecmp(tags[i],"INFO/ADR") ) flag |= B2B_INFO_ADR;
return flag;
}
+// todo: make it possible to turn off some annotations or change the defaults,
+// specifically RPB, VDB, MWU, SGB tests. It would be good to do some
+// benchmarking first to see if it's worth it.
static void list_annotations(FILE *fp)
{
fprintf(fp,
" FORMAT/ADR .. Allelic depths on the reverse strand (Number=R,Type=Integer)\n"
" FORMAT/DP .. Number of high-quality bases (Number=1,Type=Integer)\n"
" FORMAT/SP .. Phred-scaled strand bias P-value (Number=1,Type=Integer)\n"
+" FORMAT/SCR .. Number of soft-clipped reads (Number=1,Type=Integer)\n"
"\n"
"INFO annotation tags available:\n"
"\n"
" INFO/AD .. Total allelic depth (Number=R,Type=Integer)\n"
" INFO/ADF .. Total allelic depths on the forward strand (Number=R,Type=Integer)\n"
" INFO/ADR .. Total allelic depths on the reverse strand (Number=R,Type=Integer)\n"
+" INFO/SCR .. Number of soft-clipped reads (Number=1,Type=Integer)\n"
"\n");
}
" -b, --bam-list FILE list of input BAM filenames, one per line\n"
" -B, --no-BAQ disable BAQ (per-Base Alignment Quality)\n"
" -C, --adjust-MQ INT adjust mapping quality; recommended:50, disable:0 [0]\n"
-" -d, --max-depth INT max per-file depth; avoids excessive memory usage [%d]\n", mplp->max_depth);
+" -d, --max-depth INT max raw per-file depth; avoids excessive memory usage [%d]\n", mplp->max_depth);
fprintf(fp,
" -E, --redo-BAQ recalculate BAQ on the fly, ignore existing BQs\n"
" -f, --fasta-ref FILE faidx indexed reference sequence file\n"
" -o, --output FILE write output to FILE [standard output]\n"
" -O, --output-type TYPE 'b' compressed BCF; 'u' uncompressed BCF;\n"
" 'z' compressed VCF; 'v' uncompressed VCF [v]\n"
-" --threads INT number of extra output compression threads [0]\n"
+" --threads INT use multithreading with INT worker threads [0]\n"
"\n"
"SNP/INDEL genotype likelihoods options:\n"
" -e, --ext-prob INT Phred-scaled gap extension seq error probability [%d]\n", mplp->extQ);
" -P, --platforms STR comma separated list of platforms for indels [all]\n"
"\n"
"Notes: Assuming diploid individuals.\n"
+"\n"
+"Example:\n"
+" # See also http://samtools.github.io/bcftools/howtos/variant-calling.html\n"
+" bcftools mpileup -f reference.fa alignments.bam | bcftools call -mv -Ob -o calls.bcf\n"
"\n");
free(tmp_require);
mplp.record_cmd_line = 1;
mplp.n_threads = 0;
mplp.bsmpl = bam_smpl_init();
+ mplp.fmt_flag = B2B_INFO_VDB|B2B_INFO_RPB; // the default to be changed in future, see also parse_format_flag()
static const struct option lopts[] =
{
if ( mplp.gvcf && !(mplp.fmt_flag&B2B_FMT_DP) )
{
- fprintf(stderr,"[warning] The -t DP option is required with --gvcf, switching on.\n");
+ fprintf(stderr,"[warning] The -a DP option is required with --gvcf, switching on.\n");
mplp.fmt_flag |= B2B_FMT_DP;
}
if ( mplp.flag&(MPLP_BCF|MPLP_VCF|MPLP_NO_COMP) )
/* mpileup.c -- mpileup subcommand. Previously bam_plcmd.c from samtools
- Copyright (C) 2008-2017 Genome Research Ltd.
+ Copyright (C) 2008-2018 Genome Research Ltd.
Portions copyright (C) 2009-2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include <string.h>
#include <strings.h>
#include <limits.h>
+#include <inttypes.h>
#include <errno.h>
#include <sys/stat.h>
#include <getopt.h>
if (ma->conf->fai && b->core.tid >= 0) {
has_ref = mplp_get_ref(ma, b->core.tid, &ref, &ref_len);
if (has_ref && ref_len <= b->core.pos) { // exclude reads outside of the reference sequence
- fprintf(bcftools_stderr,"[%s] Skipping because %d is outside of %d [ref:%d]\n",
- __func__, b->core.pos, ref_len, b->core.tid);
+ fprintf(bcftools_stderr,"[%s] Skipping because %"PRId64" is outside of %d [ref:%d]\n",
+ __func__, (int64_t) b->core.pos, ref_len, b->core.tid);
continue;
}
} else {
// Called once per new bam added to the pileup.
// We cache sample information here so we don't have to keep recomputing this
-// on each and every pileup column.
+// on each and every pileup column. If FMT/SCR annotation is requested, a flag
+// is set to indicate the presence of a soft clip.
//
// Cd is an arbitrary block of data we can write into, which ends up in
-// the pileup structures. We stash the sample ID there.
-static int pileup_constructor(void *data, const bam1_t *b, bam_pileup_cd *cd) {
+// the pileup structures. We stash the sample ID there:
+// has_soft_clip .. cd->i & 1
+// sample_id .. cd->i >> 1
+static int pileup_constructor(void *data, const bam1_t *b, bam_pileup_cd *cd)
+{
mplp_aux_t *ma = (mplp_aux_t *)data;
- cd->i = bam_smpl_get_sample_id(ma->conf->bsmpl, ma->bam_id, (bam1_t *)b);
+ cd->i = bam_smpl_get_sample_id(ma->conf->bsmpl, ma->bam_id, (bam1_t *)b) << 1;
+ if ( ma->conf->fmt_flag & (B2B_INFO_SCR|B2B_FMT_SCR) )
+ {
+ int i;
+ for (i=0; i<b->core.n_cigar; i++)
+ {
+ int cig = bam_get_cigar(b)[i] & BAM_CIGAR_MASK;
+ if ( cig!=BAM_CSOFT_CLIP ) continue;
+ cd->i |= 1;
+ break;
+ }
+ }
return 0;
}
for (j = 0; j < n_plp[i]; ++j) // iterate over all reads available at this position
{
const bam_pileup1_t *p = plp[i] + j;
- int id = p->cd.i;
+ int id = PLP_SAMPLE_ID(p->cd.i);
if (m->n_plp[id] == m->m_plp[id])
{
m->m_plp[id] = m->m_plp[id]? m->m_plp[id]<<1 : 8;
{
if ( !conf->gvcf )
{
- if ( rec ) bcf_write1(fp, hdr, rec);
+ if ( rec && bcf_write1(fp, hdr, rec)!=0 ) error("[%s] Error: failed to write the record to %s\n", __func__,conf->output_fname?conf->output_fname:"standard output");
return;
}
if ( rec->d.allele[1][0]=='<' && rec->d.allele[1][1]=='*' && rec->d.allele[1][2]=='>' ) is_ref = 1;
}
rec = gvcf_write(conf->gvcf, fp, hdr, rec, is_ref);
- if ( rec ) bcf_write1(fp,hdr,rec);
+ if ( rec && bcf_write1(fp,hdr,rec)!=0 ) error("[%s] Error: failed to write the record to %s\n", __func__,conf->output_fname?conf->output_fname:"standard output");
}
static int mpileup_reg(mplp_conf_t *conf, uint32_t beg, uint32_t end)
while ( (ret=bam_mplp_auto(conf->iter, &tid, &pos, conf->n_plp, conf->plp)) > 0)
{
- if ( end && (pos<beg || pos>end) ) continue;
+ if ( pos<beg || pos>end ) continue;
if ( conf->bed && tid >= 0 )
{
int overlap = regidx_overlap(conf->bed, hdr->target_name[tid], pos, pos, NULL);
bcf_hdr_append(conf->bcf_hdr,"##ALT=<ID=*,Description=\"Represents allele(s) other than observed.\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=INDEL,Number=0,Type=Flag,Description=\"Indicates that the variant is an INDEL.\">");
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IDV,Number=1,Type=Integer,Description=\"Maximum number of reads supporting an indel\">");
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IMF,Number=1,Type=Float,Description=\"Maximum fraction of reads supporting an indel\">");
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IDV,Number=1,Type=Integer,Description=\"Maximum number of raw reads supporting an indel\">");
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=IMF,Number=1,Type=Float,Description=\"Maximum fraction of raw reads supporting an indel\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=DP,Number=1,Type=Integer,Description=\"Raw read depth\">");
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=VDB,Number=1,Type=Float,Description=\"Variant Distance Bias for filtering splice-site artefacts in RNA-seq data (bigger is better)\",Version=\"3\">");
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=RPB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Read Position Bias (bigger is better)\">");
+ if ( conf->fmt_flag&B2B_INFO_VDB )
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=VDB,Number=1,Type=Float,Description=\"Variant Distance Bias for filtering splice-site artefacts in RNA-seq data (bigger is better)\",Version=\"3\">");
+ if ( conf->fmt_flag&B2B_INFO_RPB )
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=RPB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Read Position Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=BQB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Base Quality Bias (bigger is better)\">");
bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=MQSB,Number=1,Type=Float,Description=\"Mann-Whitney U test of Mapping Quality vs Strand Bias (bigger is better)\">");
if ( conf->fmt_flag&B2B_FMT_SP )
bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=SP,Number=1,Type=Integer,Description=\"Phred-scaled strand bias P-value\">");
if ( conf->fmt_flag&B2B_FMT_AD )
- bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=AD,Number=R,Type=Integer,Description=\"Allelic depths\">");
+ bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=AD,Number=R,Type=Integer,Description=\"Allelic depths (high-quality bases)\">");
if ( conf->fmt_flag&B2B_FMT_ADF )
- bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADF,Number=R,Type=Integer,Description=\"Allelic depths on the forward strand\">");
+ bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADF,Number=R,Type=Integer,Description=\"Allelic depths on the forward strand (high-quality bases)\">");
if ( conf->fmt_flag&B2B_FMT_ADR )
- bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADR,Number=R,Type=Integer,Description=\"Allelic depths on the reverse strand\">");
+ bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=ADR,Number=R,Type=Integer,Description=\"Allelic depths on the reverse strand (high-quality bases)\">");
if ( conf->fmt_flag&B2B_INFO_AD )
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=AD,Number=R,Type=Integer,Description=\"Total allelic depths\">");
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=AD,Number=R,Type=Integer,Description=\"Total allelic depths (high-quality bases)\">");
if ( conf->fmt_flag&B2B_INFO_ADF )
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADF,Number=R,Type=Integer,Description=\"Total allelic depths on the forward strand\">");
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADF,Number=R,Type=Integer,Description=\"Total allelic depths on the forward strand (high-quality bases)\">");
+ if ( conf->fmt_flag&B2B_INFO_SCR )
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=SCR,Number=1,Type=Integer,Description=\"Number of soft-clipped reads (at high-quality bases)\">");
+ if ( conf->fmt_flag&B2B_FMT_SCR )
+ bcf_hdr_append(conf->bcf_hdr,"##FORMAT=<ID=SCR,Number=1,Type=Integer,Description=\"Per-sample number of soft-clipped reads (at high-quality bases)\">");
if ( conf->fmt_flag&B2B_INFO_ADR )
- bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADR,Number=R,Type=Integer,Description=\"Total allelic depths on the reverse strand\">");
+ bcf_hdr_append(conf->bcf_hdr,"##INFO=<ID=ADR,Number=R,Type=Integer,Description=\"Total allelic depths on the reverse strand (high-quality bases)\">");
if ( conf->gvcf )
gvcf_update_header(conf->gvcf, conf->bcf_hdr);
const char **smpl = bam_smpl_get_samples(conf->bsmpl, &nsmpl);
for (i=0; i<nsmpl; i++)
bcf_hdr_add_sample(conf->bcf_hdr, smpl[i]);
- bcf_hdr_write(conf->bcf_fp, conf->bcf_hdr);
+ if ( bcf_hdr_write(conf->bcf_fp, conf->bcf_hdr)!=0 ) error("[%s] Error: failed to write the header to %s\n",__func__,conf->output_fname?conf->output_fname:"standard output");
conf->bca = bcf_call_init(-1., conf->min_baseQ);
conf->bcr = (bcf_callret1_t*) calloc(nsmpl, sizeof(bcf_callret1_t));
conf->bca->min_frac = conf->min_frac;
conf->bca->min_support = conf->min_support;
conf->bca->per_sample_flt = conf->flag & MPLP_PER_SAMPLE;
+ conf->bca->fmt_flag = conf->fmt_flag;
conf->bc.bcf_hdr = conf->bcf_hdr;
conf->bc.n = nsmpl;
conf->bcr[i].ADF = conf->bc.ADF + (i+1)*B2B_MAX_ALLELES;
}
}
+ if ( conf->fmt_flag&(B2B_INFO_SCR|B2B_FMT_SCR) )
+ conf->bc.SCR = (int32_t*) malloc((nsmpl+1)*sizeof(*conf->bc.SCR));
}
// init mpileup
conf->iter = bam_mplp_init(conf->nfiles, mplp_func, (void**)conf->mplp_data);
if ( conf->flag & MPLP_SMART_OVERLAPS ) bam_mplp_init_overlaps(conf->iter);
+ fprintf(bcftools_stderr, "[%s] maximum number of reads per input file set to -d %d\n", __func__, conf->max_depth);
if ( (double)conf->max_depth * conf->nfiles > 1<<20)
fprintf(bcftools_stderr, "Warning: Potential memory hog, up to %.0fM reads in the pileup!\n", (double)conf->max_depth*conf->nfiles);
if ( (double)conf->max_depth * conf->nfiles / nsmpl < 250 )
if ( ireg++ > 0 )
{
conf->buf.l = 0;
- ksprintf(&conf->buf,"%s:%u-%u",conf->reg_itr->seq,conf->reg_itr->beg,conf->reg_itr->end);
+ ksprintf(&conf->buf,"%s:%u-%u",conf->reg_itr->seq,conf->reg_itr->beg+1,conf->reg_itr->end+1);
for (i=0; i<conf->nfiles; i++)
{
while ( regitr_loop(conf->reg_itr) );
}
else
- mpileup_reg(conf,0,0);
+ mpileup_reg(conf,0,UINT32_MAX);
flush_bcf_records(conf, conf->bcf_fp, conf->bcf_hdr, NULL);
bcf_destroy1(conf->bcf_rec);
if (conf->bcf_fp)
{
- hts_close(conf->bcf_fp);
+ if ( hts_close(conf->bcf_fp)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,conf->output_fname);
bcf_hdr_destroy(conf->bcf_hdr);
bcf_call_destroy(conf->bca);
free(conf->bc.PL);
free(conf->bc.DP4);
free(conf->bc.ADR);
free(conf->bc.ADF);
+ free(conf->bc.SCR);
free(conf->bc.fmt_arr);
free(conf->bcr);
}
files = (char**) realloc(files,nfiles*sizeof(char*));
files[nfiles-1] = strdup(buf);
}
- fclose(fh);
+ if ( fclose(fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,file_list);
if ( !nfiles )
{
fprintf(bcftools_stderr,"No files read from %s\n", file_list);
else if ( !strcasecmp(tags[i],"AD") || !strcasecmp(tags[i],"FORMAT/AD") || !strcasecmp(tags[i],"FMT/AD") ) flag |= B2B_FMT_AD;
else if ( !strcasecmp(tags[i],"ADF") || !strcasecmp(tags[i],"FORMAT/ADF") || !strcasecmp(tags[i],"FMT/ADF") ) flag |= B2B_FMT_ADF;
else if ( !strcasecmp(tags[i],"ADR") || !strcasecmp(tags[i],"FORMAT/ADR") || !strcasecmp(tags[i],"FMT/ADR") ) flag |= B2B_FMT_ADR;
+ else if ( !strcasecmp(tags[i],"SCR") || !strcasecmp(tags[i],"FORMAT/SCR") || !strcasecmp(tags[i],"FMT/SCR") ) flag |= B2B_FMT_SCR;
+ else if ( !strcasecmp(tags[i],"INFO/SCR") ) flag |= B2B_INFO_SCR;
else if ( !strcasecmp(tags[i],"INFO/AD") ) flag |= B2B_INFO_AD;
else if ( !strcasecmp(tags[i],"INFO/ADF") ) flag |= B2B_INFO_ADF;
else if ( !strcasecmp(tags[i],"INFO/ADR") ) flag |= B2B_INFO_ADR;
return flag;
}
+// todo: make it possible to turn off some annotations or change the defaults,
+// specifically RPB, VDB, MWU, SGB tests. It would be good to do some
+// benchmarking first to see if it's worth it.
static void list_annotations(FILE *fp)
{
fprintf(fp,
" FORMAT/ADR .. Allelic depths on the reverse strand (Number=R,Type=Integer)\n"
" FORMAT/DP .. Number of high-quality bases (Number=1,Type=Integer)\n"
" FORMAT/SP .. Phred-scaled strand bias P-value (Number=1,Type=Integer)\n"
+" FORMAT/SCR .. Number of soft-clipped reads (Number=1,Type=Integer)\n"
"\n"
"INFO annotation tags available:\n"
"\n"
" INFO/AD .. Total allelic depth (Number=R,Type=Integer)\n"
" INFO/ADF .. Total allelic depths on the forward strand (Number=R,Type=Integer)\n"
" INFO/ADR .. Total allelic depths on the reverse strand (Number=R,Type=Integer)\n"
+" INFO/SCR .. Number of soft-clipped reads (Number=1,Type=Integer)\n"
"\n");
}
" -b, --bam-list FILE list of input BAM filenames, one per line\n"
" -B, --no-BAQ disable BAQ (per-Base Alignment Quality)\n"
" -C, --adjust-MQ INT adjust mapping quality; recommended:50, disable:0 [0]\n"
-" -d, --max-depth INT max per-file depth; avoids excessive memory usage [%d]\n", mplp->max_depth);
+" -d, --max-depth INT max raw per-file depth; avoids excessive memory usage [%d]\n", mplp->max_depth);
fprintf(fp,
" -E, --redo-BAQ recalculate BAQ on the fly, ignore existing BQs\n"
" -f, --fasta-ref FILE faidx indexed reference sequence file\n"
" -o, --output FILE write output to FILE [standard output]\n"
" -O, --output-type TYPE 'b' compressed BCF; 'u' uncompressed BCF;\n"
" 'z' compressed VCF; 'v' uncompressed VCF [v]\n"
-" --threads INT number of extra output compression threads [0]\n"
+" --threads INT use multithreading with INT worker threads [0]\n"
"\n"
"SNP/INDEL genotype likelihoods options:\n"
" -e, --ext-prob INT Phred-scaled gap extension seq error probability [%d]\n", mplp->extQ);
" -P, --platforms STR comma separated list of platforms for indels [all]\n"
"\n"
"Notes: Assuming diploid individuals.\n"
+"\n"
+"Example:\n"
+" # See also http://samtools.github.io/bcftools/howtos/variant-calling.html\n"
+" bcftools mpileup -f reference.fa alignments.bam | bcftools call -mv -Ob -o calls.bcf\n"
"\n");
free(tmp_require);
mplp.record_cmd_line = 1;
mplp.n_threads = 0;
mplp.bsmpl = bam_smpl_init();
+ mplp.fmt_flag = B2B_INFO_VDB|B2B_INFO_RPB; // the default to be changed in future, see also parse_format_flag()
static const struct option lopts[] =
{
if ( mplp.gvcf && !(mplp.fmt_flag&B2B_FMT_DP) )
{
- fprintf(bcftools_stderr,"[warning] The -t DP option is required with --gvcf, switching on.\n");
+ fprintf(bcftools_stderr,"[warning] The -a DP option is required with --gvcf, switching on.\n");
mplp.fmt_flag |= B2B_FMT_DP;
}
if ( mplp.flag&(MPLP_BCF|MPLP_VCF|MPLP_NO_COMP) )
+++ /dev/null
-/* plugins/GTisec.c -- collect genotype intersection counts of all possible
- subsets of the present samples and output in banker's
- sequence order (in this sequence, the number of contained
- samples increases monotonically, a property that is e.g.
- useful for programatically creating plotting files for the
- R package VennDiagram or the plotting tool circos from the
- counts, as in the command line tools bankers2VennDiagram and
- bankers2circos at htpps://github.com/dlaehnemann/bankers2)
-
- Copyright (C) 2016 Computational Biology of Infection Research,
- Helmholtz Centre for Infection Research, Braunschweig,
- Germany
-
- Author: David Laehnemann <david.laehnemann@helmholtz-hzi.de>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <unistd.h>
-#include <inttypes.h>
-
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/khash.h>
-KHASH_MAP_INIT_INT(gts2smps, uint32_t)
-
-#include "bcftools.h"
-
-/*!
- * Flag definitions for args.flag
- */
-#define MISSING (1<<0)
-#define VERBOSE (1<<1)
-#define SMPORDER (1<<2)
-
-typedef struct _args_t
-{
- bcf_srs_t *file; /*! multi-sample VCF file */
- bcf_hdr_t *hdr; /*! VCF file header */
- FILE *out; /*! output file pointer */
- int nsmp; /*! number of samples, can be determined from header but is needed in multiple contexts */
- int nsmpp2; /*! 2^(nsmp) (is needed multiple times) */
- int *gt_arr; /*! temporary array, to store GTs of current line/record */
- int ngt_arr; /*! hold the number of current GT array entries */
- uint32_t *bankers; /*! array to store banker's sequence for all possible sample subsets for
- programmatic indexing into smp_is for output printing, e.g. for three
- samples A, B and C this would be the following order:
- [ C, B, A, CB, CA, BA, CBA ]
- [ 100, 010, 001, 110, 101, 011, 111 ]
- */
- uint64_t *quick; /*! array to store n choose k lookup table of choose() function */
- uint8_t flag; /*! several flags, for positions see above*/
- uint64_t *missing_gts; /*! array to count missing genotypes of each sample */
- uint64_t *smp_is; /*! array to track all possible intersections between
- samples, with each bit in the index integer belonging to one
- sample. E.g. for three samples A, B and C, count would be in
- the following order:
- [ A, B, AB, C, AC, BC, ABC ]
- [ 001, 010, 011, 100, 101, 110, 111 ]
- */
-}
-args_t;
-
-static args_t args;
-uint32_t compute_bankers(unsigned long a);
-
-const char *about(void)
-{
- return "Count genotype intersections across all possible sample subsets in a vcf file.\n";
-}
-
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Count genotype intersections across all possible sample subsets in a vcf file.\n"
- "Usage: bcftools +GTisec <multisample.bcf/.vcf.gz> [General Options] -- [Plugin Options] \n"
- "\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -m, --missing if set, include count of missing genotypes per sample in output\n"
- " -v, --verbose if set, annotate count rows with corresponding sample subset lists\n"
- " -H, --human-readable if set, create human readable output; i.e. sort output by sample and\n"
- " print each subset's intersection count once for each sample contained\n"
- " in the subset; implies verbose output (-v)\n"
- "\n"
- "Example:\n"
- " bcftools +GTisec in.vcf -- -v # for verbose output\n"
- " bcftools +GTisec in.vcf -- -H # for human readable output\n"
- "\n";
-}
-
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- memset(&args,0,sizeof(args_t));
- args.flag = 0;
-
- static struct option loptions[] =
- {
- {"help", no_argument, 0,'h'},
- {"missing", no_argument, 0,'m'},
- {"verbose", no_argument, 0,'v'},
- {"human-readable", no_argument, 0,'H'},
- {0,0,0,0}
- };
-
- int c;
- while ((c = getopt_long(argc, argv, "?mvHh",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'm': args.flag |= MISSING; break;
- case 'v': args.flag |= VERBOSE; break;
- case 'H': args.flag |= ( SMPORDER | VERBOSE ); break;
- case 'h': usage(); break;
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( optind != argc ) usage(); // too many files given
-
-
- args.hdr = in;
-
- if ( !bcf_hdr_nsamples(args.hdr) )
- {
- error("No samples in input file.\n");
- }
-
- args.nsmp = bcf_hdr_nsamples(args.hdr);
- if ( args.nsmp > 32 ) error("Too many samples. A maximum of 32 is supported.\n");
- args.nsmpp2 = pow( 2, args.nsmp);
- args.bankers = (uint32_t*) calloc( args.nsmpp2, sizeof(uint32_t) );
- args.quick = (uint64_t*) calloc((args.nsmp * (args.nsmp + 1)) / 4, sizeof(unsigned long));
- if ( args.flag & MISSING ) args.missing_gts = (uint64_t*) calloc( args.nsmp, sizeof(uint64_t));
- args.smp_is = (uint64_t*) calloc( args.nsmpp2, sizeof(uint64_t));
- if ( bcf_hdr_id2int(args.hdr, BCF_DT_ID, "GT")<0 ) error("[E::%s] GT not present in the header\n", __func__);
-
- args.gt_arr = NULL;
- args.ngt_arr = 0;
-
- args.out = stdout;
-
- /*! Header printing */
- FILE *fp = args.out;
- fprintf(fp, "# This file was produced by bcftools +GTisec (%s+htslib-%s)\n", bcftools_version(), hts_version());
- fprintf(fp, "# The command line was:\tbcftools +GTisec %s ", argv[0]);
- int i;
- for (i=1; i < argc; i++)
- {
- fprintf(fp, " %s", argv[i]);
- }
- fprintf(fp,"\n");
- fprintf(fp,"# This file can be used as input to the subset plotting tools at:\n"
- "# https://github.com/dlaehnemann/bankers2\n");
- fprintf(fp,"# Genotype intersections across samples:\n");
- fprintf(fp,"@SMPS");
- for (i = args.nsmp-1; i >= 0; i--)
- {
- fprintf(fp," %s", bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, i));
- }
- fprintf(fp,"\n");
- if ( args.flag & MISSING )
- {
- if ( args.flag & SMPORDER )
- {
- fprintf(fp, "# The first line of each sample contains its count of missing genotypes, with a '-' appended\n"
- "# to the sample name.\n");
- }
- else
- {
- fprintf(fp, "# The first %i lines contain the counts for missing values of each sample in the order provided\n"
- "# in the SMPS-line above. Intersection counts only start afterwards.\n", args.nsmp);
- }
- }
- if ( args.flag & SMPORDER )
- {
- fprintf(fp, "# Human readable output (-H) was requested. Subset intersection counts are therefore sorted by\n"
- "# sample and repeated for each contained sample. For each sample, counts are in banker's \n"
- "# sequence order regarding all other samples.\n");
- }
- else
- {
- fprintf(fp, "# Subset intersection counts are in global banker's sequence order.\n");
- if ( args.nsmp > 2 )
- {
- fprintf(fp, "# After exclusive sample counts in order of the SMPS-line, banker's sequence continues with:\n"
- "# %s,%s %s,%s ...\n",
- bcf_hdr_int2id(in, BCF_DT_SAMPLE, args.nsmp-1 ),
- bcf_hdr_int2id(in, BCF_DT_SAMPLE, args.nsmp-2 ),
- bcf_hdr_int2id(in, BCF_DT_SAMPLE, args.nsmp-1 ),
- bcf_hdr_int2id(in, BCF_DT_SAMPLE, args.nsmp-3 )
- );
- }
- }
- if (args.flag & VERBOSE )
- {
- fprintf(fp,"# [1] Number of shared non-ref genotypes \t[2] Samples sharing non-ref genotype (GT)\n");
- }
- else
- {
- fprintf(fp,"# [1] Number of shared non-ref genotypes\n");
- }
-
- /* Compute banker's sequence for following printing by sample and
- * with increasing subset size.
- */
- uint32_t j;
- for ( j = 0; j < args.nsmpp2; j++ )
- {
- args.bankers[j] = compute_bankers(j);
- }
-
- return 1;
-}
-
-
-/* ADAPTED CODE FROM CORIN LAWSON (START)
- * https://github.com/au-phiware/bankers/blob/master/c/bankers.c
- * who implemented ideas of Eric Burnett:
- * http://www.thelowlyprogrammer.com/2010/04/indexing-and-enumerating-subsets-of.html
- */
-
-/*
- * Compute the binomial coefficient of `n choose k'.
- * Use the fact that binom(n, k) = binom(n, n - k).
- * Use a lookup table (triangle, actually) for speed.
- * Otherwise it's dumb (heart) recursion.
- * Added relative to Corin Lawson:
- * * Passing in of sample number through pointer to args struct
- * * Make quick lookup table external to keep it persistent with clean allocation
- * and freeing
- */
-uint64_t choose(unsigned int n, unsigned int k) {
- if (n == 0)
- return 0;
- if (n == k || k == 0)
- return 1;
- if (k > n / 2)
- k = n - k;
-
- unsigned int i = (n * (n - 1)) / 4 + k - 1;
- if (args.quick[i] == 0)
- args.quick[i] = choose(n - 1, k - 1) + choose(n - 1, k);
-
- return args.quick[i];
-}
-
-/*
- * Returns the Banker's number at the specified position, a.
- * Derived from the recursive bit flip method.
- * Added relative to Corin Lawson:
- * * Uses same lookup table solution as choose function, just
- * maintained externally to persist across separate function calls.
- * * Uses bitwise symmetry of banker's sequence to use bitwise inversion
- * instead of recursive bit flip for second half of sequence.
- */
-uint32_t compute_bankers(unsigned long a)
-{
- if (a == 0)
- return 0;
-
- if ( args.bankers[a] == 0 )
- {
- if ( a >= (args.nsmpp2 / 2) )
- return args.bankers[a] = ( compute_bankers(args.nsmpp2 - (a+1)) ^ (args.nsmpp2 - 1) ); // use bitwise symmetry of bankers sequence
- unsigned int c = 0;
- uint32_t n = args.nsmp;
- uint64_t e = a, binom;
- binom = choose(n, c);
- do {
- e -= binom;
- } while ((binom = choose(n, ++c)) <= e);
-
- do {
- if (e == 0 || (binom = choose(n - 1, c - 1)) > e)
- c--, args.bankers[a] |= 1;
- else
- e -= binom;
- } while (--n && c && ((args.bankers[a] <<= 1) || 1));
- args.bankers[a] <<= n;
- }
-
- return args.bankers[a];
-}
-
-// ADAPTED CODE FROM CORIN LAWSON END
-
-
-/*
- * GT field (genotype) comparison function.
- */
-bcf1_t *process(bcf1_t *rec)
-{
- uint64_t i;
- bcf_unpack(rec, BCF_UN_FMT); // unpack the Format fields, including the GT field
- int gte_smp = 0; // number GT array entries per sample (should be 2, one entry per allele)
- if ( (gte_smp = bcf_get_genotypes(args.hdr, rec, &(args.gt_arr), &(args.ngt_arr) ) ) <= 0 )
- {
- error("GT not present at %s: %d\n", args.hdr->id[BCF_DT_CTG][rec->rid].key, rec->pos+1);
- }
-
- gte_smp /= args.nsmp; // divide total number of genotypes array entries (= args.ngt_arr) by number of samples
- int ret;
-
- // stick all genotypes in a hash as keys and store up to 32 samples in a corresponding flag as its value
- khiter_t bucket;
- khash_t(gts2smps) *gts = kh_init(gts2smps); // create hash
- for ( i = 0; i < args.nsmp; i++ )
- {
- int *gt_ptr = args.gt_arr + gte_smp * i;
-
- if (bcf_gt_is_missing(gt_ptr[0]) || ( gte_smp == 2 && bcf_gt_is_missing(gt_ptr[1]) ) )
- {
- if ( args.flag & MISSING ) args.missing_gts[i]++; // count missing genotypes, if requested
- continue; // don't do anything else for missing genotypes, their "sharing" gives no info...
- }
-
- int a = bcf_gt_allele(gt_ptr[0]);
- int b;
- if ( gte_smp == 2 ) // two entries available per sample, padded with missing values for haploid genotypes
- {
- b = bcf_gt_allele(gt_ptr[1]);
- }
- else if (gte_smp == 1 ) // use missing value for second entry in hash key generation below, if only one is available
- {
- b = bcf_gt_allele(bcf_int32_vector_end);
- }
- else
- {
- error("gtisec does not support ploidy higher than 2.\n");
- }
-
- int idx = bcf_alleles2gt(a,b); // generate genotype specific hash key
-
- bucket = kh_get(gts2smps, gts, idx); // get the genotype's hash bucket
-
- if ( bucket == kh_end(gts) ) { // means that key does not exist
- bucket = kh_put(gts2smps, gts, idx, &ret); // create bucket with genotype index as key and return its iterator
- kh_val(gts, bucket) = 0; // initialize the bucket with all sample bits unset
- }
- kh_value(gts, bucket) |= (1<<i); // set the sample's bit to 1 in this genotype's bucket
- }
-
- // iterate over genotypes and for each genotype increment the appropriate smp_is entry
- for ( bucket = kh_begin(gts); bucket != kh_end(gts); ++bucket ) // iterate over all genotypes at this position
- {
- if ( kh_exist(gts, bucket) ) // for existing genotype buckets
- {
- uint32_t s = kh_val(gts, bucket); // get the 32 bit flag
- args.smp_is[s]++; // add to the corresponding subset
- }
- }
- kh_destroy(gts2smps, gts); // destroy hash
-
- return NULL;
-}
-
-void destroy(void)
-{
- int32_t i;
- int s;
-
- FILE *fp = args.out;
-
- /* Printing to File */
- if ( args.flag & SMPORDER )
- {
- /* Iterate over samples, printing out all subsets including
- * the current sample, with the current sample first. This
- * includes multiple printouts of the same sample but makes
- * output more readable and is also needed for circos files
- * printing.
- */
- for ( s = args.nsmp-1; s >= 0; s--)
- {
- if ( args.flag & MISSING ) // if missing genotype counts are requested, print them to standard output
- {
- fprintf(fp, "%"PRIu64"\t%s-\n", args.missing_gts[s], bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, s));
- }
- for ( i = 1; i < args.nsmpp2; i++ )
- {
- if ( (args.bankers[i]>>s) & 1 )
- {
- fprintf(fp, "%"PRIu64"\t", args.smp_is[ args.bankers[i] ]); // print out count of genotypes shared by samples in current banker's sequence position
- int j;
- /* Print sample list */
- fprintf(fp, "%s", bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, s)); // print current sample first
- for ( j = args.nsmp-1; j >= 0; j-- )
- {
- if ( (args.bankers[i] ^ (1<<s)) & (1<<j) ) // exclude current sample from printing again
- {
- fprintf(fp, ",%s", bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, j ) ); // print out sample list, starting with our current major sample
- }
- }
- fprintf(fp, "\n" );
- }
- }
- }
- }
- else if ( args.flag & VERBOSE )
- {
- if ( args.flag & MISSING ) // if missing genotype counts are requested, print them to standard output
- {
- for ( s = args.nsmp-1; s >= 0; s--)
- {
- fprintf(fp, "%"PRIu64"\t%s-\n", args.missing_gts[s], bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, s));
- }
- }
- for ( i = 1; i < args.nsmpp2; i++ )
- {
- fprintf(fp, "%"PRIu64"\t", args.smp_is[ args.bankers[i] ]); // print out count of genotypes shared by samples in current banker's sequence position
- int j = 0;
- for ( s = args.nsmp-1; s >= 0; s--)
- {
- if ( (args.bankers[i]>>s) & 1 )
- {
- fprintf(fp, "%s%s", j ? "," : "", bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, s) ); // samples in specified order
- j = 1;
- }
- }
- fprintf(fp, "\n" );
- }
- }
- else
- {
- if ( args.flag & MISSING ) // if missing genotype counts are requested, print them to standard output
- {
- for ( s = args.nsmp-1; s >= 0; s--)
- {
- fprintf(fp, "%"PRIu64"\n", args.missing_gts[s]);
- }
- }
- for ( i = 1; i < args.nsmpp2; i++ )
- {
- fprintf(fp, "%"PRIu64"\n", args.smp_is[ args.bankers[i] ]); // print out count of genotypes shared by samples in current banker's sequence position
- }
- }
- fclose(fp);
-
- /* freeing up args */
- free(args.gt_arr);
- free(args.bankers);
- free(args.quick);
- if (args.flag & MISSING) free(args.missing_gts);
- free(args.smp_is);
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugins/GTisec.c -- collect genotype intersection counts of all possible
- subsets of the present samples and output in banker's
- sequence order (in this sequence, the number of contained
- samples increases monotonically, a property that is e.g.
- useful for programatically creating plotting files for the
- R package VennDiagram or the plotting tool circos from the
- counts, as in the command line tools bankers2VennDiagram and
- bankers2circos at htpps://github.com/dlaehnemann/bankers2)
-
- Copyright (C) 2016 Computational Biology of Infection Research,
- Helmholtz Centre for Infection Research, Braunschweig,
- Germany
-
- Author: David Laehnemann <david.laehnemann@helmholtz-hzi.de>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <unistd.h>
-#include <inttypes.h>
-
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/khash.h>
-KHASH_MAP_INIT_INT(gts2smps, uint32_t)
-
-#include "bcftools.h"
-
-/*!
- * Flag definitions for args.flag
- */
-#define MISSING (1<<0)
-#define VERBOSE (1<<1)
-#define SMPORDER (1<<2)
-
-typedef struct _args_t
-{
- bcf_srs_t *file; /*! multi-sample VCF file */
- bcf_hdr_t *hdr; /*! VCF file header */
- FILE *out; /*! output file pointer */
- int nsmp; /*! number of samples, can be determined from header but is needed in multiple contexts */
- int nsmpp2; /*! 2^(nsmp) (is needed multiple times) */
- int *gt_arr; /*! temporary array, to store GTs of current line/record */
- int ngt_arr; /*! hold the number of current GT array entries */
- uint32_t *bankers; /*! array to store banker's sequence for all possible sample subsets for
- programmatic indexing into smp_is for output printing, e.g. for three
- samples A, B and C this would be the following order:
- [ C, B, A, CB, CA, BA, CBA ]
- [ 100, 010, 001, 110, 101, 011, 111 ]
- */
- uint64_t *quick; /*! array to store n choose k lookup table of choose() function */
- uint8_t flag; /*! several flags, for positions see above*/
- uint64_t *missing_gts; /*! array to count missing genotypes of each sample */
- uint64_t *smp_is; /*! array to track all possible intersections between
- samples, with each bit in the index integer belonging to one
- sample. E.g. for three samples A, B and C, count would be in
- the following order:
- [ A, B, AB, C, AC, BC, ABC ]
- [ 001, 010, 011, 100, 101, 110, 111 ]
- */
-}
-args_t;
-
-static args_t args;
-uint32_t compute_bankers(unsigned long a);
-
-const char *about(void)
-{
- return "Count genotype intersections across all possible sample subsets in a vcf file.\n";
-}
-
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Count genotype intersections across all possible sample subsets in a vcf file.\n"
- "Usage: bcftools +GTisec <multisample.bcf/.vcf.gz> [General Options] -- [Plugin Options] \n"
- "\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -m, --missing if set, include count of missing genotypes per sample in output\n"
- " -v, --verbose if set, annotate count rows with corresponding sample subset lists\n"
- " -H, --human-readable if set, create human readable output; i.e. sort output by sample and\n"
- " print each subset's intersection count once for each sample contained\n"
- " in the subset; implies verbose output (-v)\n"
- "\n"
- "Example:\n"
- " bcftools +GTisec in.vcf -- -v # for verbose output\n"
- " bcftools +GTisec in.vcf -- -H # for human readable output\n"
- "\n";
-}
-
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- memset(&args,0,sizeof(args_t));
- args.flag = 0;
-
- static struct option loptions[] =
- {
- {"help", no_argument, 0,'h'},
- {"missing", no_argument, 0,'m'},
- {"verbose", no_argument, 0,'v'},
- {"human-readable", no_argument, 0,'H'},
- {0,0,0,0}
- };
-
- int c;
- while ((c = getopt_long(argc, argv, "?mvHh",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'm': args.flag |= MISSING; break;
- case 'v': args.flag |= VERBOSE; break;
- case 'H': args.flag |= ( SMPORDER | VERBOSE ); break;
- case 'h': usage(); break;
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( optind != argc ) usage(); // too many files given
-
-
- args.hdr = in;
-
- if ( !bcf_hdr_nsamples(args.hdr) )
- {
- error("No samples in input file.\n");
- }
-
- args.nsmp = bcf_hdr_nsamples(args.hdr);
- if ( args.nsmp > 32 ) error("Too many samples. A maximum of 32 is supported.\n");
- args.nsmpp2 = pow( 2, args.nsmp);
- args.bankers = (uint32_t*) calloc( args.nsmpp2, sizeof(uint32_t) );
- args.quick = (uint64_t*) calloc((args.nsmp * (args.nsmp + 1)) / 4, sizeof(unsigned long));
- if ( args.flag & MISSING ) args.missing_gts = (uint64_t*) calloc( args.nsmp, sizeof(uint64_t));
- args.smp_is = (uint64_t*) calloc( args.nsmpp2, sizeof(uint64_t));
- if ( bcf_hdr_id2int(args.hdr, BCF_DT_ID, "GT")<0 ) error("[E::%s] GT not present in the header\n", __func__);
-
- args.gt_arr = NULL;
- args.ngt_arr = 0;
-
- args.out = bcftools_stdout;
-
- /*! Header printing */
- FILE *fp = args.out;
- fprintf(fp, "# This file was produced by bcftools +GTisec (%s+htslib-%s)\n", bcftools_version(), hts_version());
- fprintf(fp, "# The command line was:\tbcftools +GTisec %s ", argv[0]);
- int i;
- for (i=1; i < argc; i++)
- {
- fprintf(fp, " %s", argv[i]);
- }
- fprintf(fp,"\n");
- fprintf(fp,"# This file can be used as input to the subset plotting tools at:\n"
- "# https://github.com/dlaehnemann/bankers2\n");
- fprintf(fp,"# Genotype intersections across samples:\n");
- fprintf(fp,"@SMPS");
- for (i = args.nsmp-1; i >= 0; i--)
- {
- fprintf(fp," %s", bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, i));
- }
- fprintf(fp,"\n");
- if ( args.flag & MISSING )
- {
- if ( args.flag & SMPORDER )
- {
- fprintf(fp, "# The first line of each sample contains its count of missing genotypes, with a '-' appended\n"
- "# to the sample name.\n");
- }
- else
- {
- fprintf(fp, "# The first %i lines contain the counts for missing values of each sample in the order provided\n"
- "# in the SMPS-line above. Intersection counts only start afterwards.\n", args.nsmp);
- }
- }
- if ( args.flag & SMPORDER )
- {
- fprintf(fp, "# Human readable output (-H) was requested. Subset intersection counts are therefore sorted by\n"
- "# sample and repeated for each contained sample. For each sample, counts are in banker's \n"
- "# sequence order regarding all other samples.\n");
- }
- else
- {
- fprintf(fp, "# Subset intersection counts are in global banker's sequence order.\n");
- if ( args.nsmp > 2 )
- {
- fprintf(fp, "# After exclusive sample counts in order of the SMPS-line, banker's sequence continues with:\n"
- "# %s,%s %s,%s ...\n",
- bcf_hdr_int2id(in, BCF_DT_SAMPLE, args.nsmp-1 ),
- bcf_hdr_int2id(in, BCF_DT_SAMPLE, args.nsmp-2 ),
- bcf_hdr_int2id(in, BCF_DT_SAMPLE, args.nsmp-1 ),
- bcf_hdr_int2id(in, BCF_DT_SAMPLE, args.nsmp-3 )
- );
- }
- }
- if (args.flag & VERBOSE )
- {
- fprintf(fp,"# [1] Number of shared non-ref genotypes \t[2] Samples sharing non-ref genotype (GT)\n");
- }
- else
- {
- fprintf(fp,"# [1] Number of shared non-ref genotypes\n");
- }
-
- /* Compute banker's sequence for following printing by sample and
- * with increasing subset size.
- */
- uint32_t j;
- for ( j = 0; j < args.nsmpp2; j++ )
- {
- args.bankers[j] = compute_bankers(j);
- }
-
- return 1;
-}
-
-
-/* ADAPTED CODE FROM CORIN LAWSON (START)
- * https://github.com/au-phiware/bankers/blob/master/c/bankers.c
- * who implemented ideas of Eric Burnett:
- * http://www.thelowlyprogrammer.com/2010/04/indexing-and-enumerating-subsets-of.html
- */
-
-/*
- * Compute the binomial coefficient of `n choose k'.
- * Use the fact that binom(n, k) = binom(n, n - k).
- * Use a lookup table (triangle, actually) for speed.
- * Otherwise it's dumb (heart) recursion.
- * Added relative to Corin Lawson:
- * * Passing in of sample number through pointer to args struct
- * * Make quick lookup table external to keep it persistent with clean allocation
- * and freeing
- */
-uint64_t choose(unsigned int n, unsigned int k) {
- if (n == 0)
- return 0;
- if (n == k || k == 0)
- return 1;
- if (k > n / 2)
- k = n - k;
-
- unsigned int i = (n * (n - 1)) / 4 + k - 1;
- if (args.quick[i] == 0)
- args.quick[i] = choose(n - 1, k - 1) + choose(n - 1, k);
-
- return args.quick[i];
-}
-
-/*
- * Returns the Banker's number at the specified position, a.
- * Derived from the recursive bit flip method.
- * Added relative to Corin Lawson:
- * * Uses same lookup table solution as choose function, just
- * maintained externally to persist across separate function calls.
- * * Uses bitwise symmetry of banker's sequence to use bitwise inversion
- * instead of recursive bit flip for second half of sequence.
- */
-uint32_t compute_bankers(unsigned long a)
-{
- if (a == 0)
- return 0;
-
- if ( args.bankers[a] == 0 )
- {
- if ( a >= (args.nsmpp2 / 2) )
- return args.bankers[a] = ( compute_bankers(args.nsmpp2 - (a+1)) ^ (args.nsmpp2 - 1) ); // use bitwise symmetry of bankers sequence
- unsigned int c = 0;
- uint32_t n = args.nsmp;
- uint64_t e = a, binom;
- binom = choose(n, c);
- do {
- e -= binom;
- } while ((binom = choose(n, ++c)) <= e);
-
- do {
- if (e == 0 || (binom = choose(n - 1, c - 1)) > e)
- c--, args.bankers[a] |= 1;
- else
- e -= binom;
- } while (--n && c && ((args.bankers[a] <<= 1) || 1));
- args.bankers[a] <<= n;
- }
-
- return args.bankers[a];
-}
-
-// ADAPTED CODE FROM CORIN LAWSON END
-
-
-/*
- * GT field (genotype) comparison function.
- */
-bcf1_t *process(bcf1_t *rec)
-{
- uint64_t i;
- bcf_unpack(rec, BCF_UN_FMT); // unpack the Format fields, including the GT field
- int gte_smp = 0; // number GT array entries per sample (should be 2, one entry per allele)
- if ( (gte_smp = bcf_get_genotypes(args.hdr, rec, &(args.gt_arr), &(args.ngt_arr) ) ) <= 0 )
- {
- error("GT not present at %s: %d\n", args.hdr->id[BCF_DT_CTG][rec->rid].key, rec->pos+1);
- }
-
- gte_smp /= args.nsmp; // divide total number of genotypes array entries (= args.ngt_arr) by number of samples
- int ret;
-
- // stick all genotypes in a hash as keys and store up to 32 samples in a corresponding flag as its value
- khiter_t bucket;
- khash_t(gts2smps) *gts = kh_init(gts2smps); // create hash
- for ( i = 0; i < args.nsmp; i++ )
- {
- int *gt_ptr = args.gt_arr + gte_smp * i;
-
- if (bcf_gt_is_missing(gt_ptr[0]) || ( gte_smp == 2 && bcf_gt_is_missing(gt_ptr[1]) ) )
- {
- if ( args.flag & MISSING ) args.missing_gts[i]++; // count missing genotypes, if requested
- continue; // don't do anything else for missing genotypes, their "sharing" gives no info...
- }
-
- int a = bcf_gt_allele(gt_ptr[0]);
- int b;
- if ( gte_smp == 2 ) // two entries available per sample, padded with missing values for haploid genotypes
- {
- b = bcf_gt_allele(gt_ptr[1]);
- }
- else if (gte_smp == 1 ) // use missing value for second entry in hash key generation below, if only one is available
- {
- b = bcf_gt_allele(bcf_int32_vector_end);
- }
- else
- {
- error("gtisec does not support ploidy higher than 2.\n");
- }
-
- int idx = bcf_alleles2gt(a,b); // generate genotype specific hash key
-
- bucket = kh_get(gts2smps, gts, idx); // get the genotype's hash bucket
-
- if ( bucket == kh_end(gts) ) { // means that key does not exist
- bucket = kh_put(gts2smps, gts, idx, &ret); // create bucket with genotype index as key and return its iterator
- kh_val(gts, bucket) = 0; // initialize the bucket with all sample bits unset
- }
- kh_value(gts, bucket) |= (1<<i); // set the sample's bit to 1 in this genotype's bucket
- }
-
- // iterate over genotypes and for each genotype increment the appropriate smp_is entry
- for ( bucket = kh_begin(gts); bucket != kh_end(gts); ++bucket ) // iterate over all genotypes at this position
- {
- if ( kh_exist(gts, bucket) ) // for existing genotype buckets
- {
- uint32_t s = kh_val(gts, bucket); // get the 32 bit flag
- args.smp_is[s]++; // add to the corresponding subset
- }
- }
- kh_destroy(gts2smps, gts); // destroy hash
-
- return NULL;
-}
-
-void destroy(void)
-{
- int32_t i;
- int s;
-
- FILE *fp = args.out;
-
- /* Printing to File */
- if ( args.flag & SMPORDER )
- {
- /* Iterate over samples, printing out all subsets including
- * the current sample, with the current sample first. This
- * includes multiple printouts of the same sample but makes
- * output more readable and is also needed for circos files
- * printing.
- */
- for ( s = args.nsmp-1; s >= 0; s--)
- {
- if ( args.flag & MISSING ) // if missing genotype counts are requested, print them to standard output
- {
- fprintf(fp, "%"PRIu64"\t%s-\n", args.missing_gts[s], bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, s));
- }
- for ( i = 1; i < args.nsmpp2; i++ )
- {
- if ( (args.bankers[i]>>s) & 1 )
- {
- fprintf(fp, "%"PRIu64"\t", args.smp_is[ args.bankers[i] ]); // print out count of genotypes shared by samples in current banker's sequence position
- int j;
- /* Print sample list */
- fprintf(fp, "%s", bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, s)); // print current sample first
- for ( j = args.nsmp-1; j >= 0; j-- )
- {
- if ( (args.bankers[i] ^ (1<<s)) & (1<<j) ) // exclude current sample from printing again
- {
- fprintf(fp, ",%s", bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, j ) ); // print out sample list, starting with our current major sample
- }
- }
- fprintf(fp, "\n" );
- }
- }
- }
- }
- else if ( args.flag & VERBOSE )
- {
- if ( args.flag & MISSING ) // if missing genotype counts are requested, print them to standard output
- {
- for ( s = args.nsmp-1; s >= 0; s--)
- {
- fprintf(fp, "%"PRIu64"\t%s-\n", args.missing_gts[s], bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, s));
- }
- }
- for ( i = 1; i < args.nsmpp2; i++ )
- {
- fprintf(fp, "%"PRIu64"\t", args.smp_is[ args.bankers[i] ]); // print out count of genotypes shared by samples in current banker's sequence position
- int j = 0;
- for ( s = args.nsmp-1; s >= 0; s--)
- {
- if ( (args.bankers[i]>>s) & 1 )
- {
- fprintf(fp, "%s%s", j ? "," : "", bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, s) ); // samples in specified order
- j = 1;
- }
- }
- fprintf(fp, "\n" );
- }
- }
- else
- {
- if ( args.flag & MISSING ) // if missing genotype counts are requested, print them to standard output
- {
- for ( s = args.nsmp-1; s >= 0; s--)
- {
- fprintf(fp, "%"PRIu64"\n", args.missing_gts[s]);
- }
- }
- for ( i = 1; i < args.nsmpp2; i++ )
- {
- fprintf(fp, "%"PRIu64"\n", args.smp_is[ args.bankers[i] ]); // print out count of genotypes shared by samples in current banker's sequence position
- }
- }
- fclose(fp);
-
- /* freeing up args */
- free(args.gt_arr);
- free(args.bankers);
- free(args.quick);
- if (args.flag & MISSING) free(args.missing_gts);
- free(args.smp_is);
-}
+++ /dev/null
-/* plugins/GTsubset.c -- output only positions where the selected samples exclusively
- share a genotype, i.e. all selected samples must have the same
- genotype (including both alleles) and none of the unselected
- samples can have the same genotype
-
- Copyright (C) 2016 Computational Biology of Infection Research,
- Helmholtz Centre for Infection Research, Braunschweig,
- Germany
-
- Author: David Laehnemann <david.laehnemann@hhu.de>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <unistd.h>
-#include <inttypes.h>
-
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-
-#include "bcftools.h"
-
-typedef struct _args_t
-{
- bcf_hdr_t *hdr; /*! VCF file header */
- int *gt_arr; /*! temporary array, to store GTs of current line/record */
- int ngt_arr; /*! hold the number of current GT array entries */
- int nsmp; /*! number of samples, can be determined from header but is needed in multiple contexts */
- int n_sel_smps; /*! number of selected samples who should exclusively share genotypes */
- int *selected_smps; /*! pointer to start of array containing 1 at indices corresponding to selected samples in header dict and 0 at others*/
-}
-args_t;
-
-static args_t args;
-
-const char *about(void)
-{
- return "Output only sites where the requested samples all exclusively share a genotype (GT).\n";
-}
-
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Output only sites where the requested samples all exclusively share a genotype (GT), i.e.\n"
- " all selected samples must have the same GT, while non of the others can have it.\n"
- "Usage: bcftools +GTsubset <multisample.bcf/.vcf.gz> [General Options] -- [Plugin Options] \n"
- "\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -s,--sample-list comma-separated list of samples; only those sites where all of these\n"
- " samples exclusively share their genotype are given as output\n"
- "\n"
- "Example:\n"
- " bcftools +GTsubset in.vcf -- -s SMP1,SMP2 \n"
- "\n";
-}
-
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- memset(&args,0,sizeof(args_t));
-
- int i;
-
- static struct option loptions[] =
- {
- {"help", no_argument, 0,'h'},
- {"sample-list", required_argument, 0,'s'},
- {0,0,0,0}
- };
-
- char **smps_strs = NULL;
-
- int c;
- while ((c = getopt_long(argc, argv, "?s:h",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 's': smps_strs = hts_readlist(optarg,0,&(args.n_sel_smps));
- if ( args.n_sel_smps == 0 )
- {
- fprintf(stderr, "Sample specification not valid.\n");
- error("%s", usage());
- }
- break;
- case 'h': usage(); break;
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( optind != argc ) usage(); // too many files given
-
- args.hdr = bcf_hdr_dup(in);
-
- // Samples parsing from header and input option
- if ( !bcf_hdr_nsamples(args.hdr) )
- {
- error("No samples in input file.\n");
- }
- args.nsmp = bcf_hdr_nsamples(args.hdr);
- args.selected_smps = (int*) calloc(args.nsmp,sizeof(int));
- for ( i = 0; i < args.n_sel_smps; i++ )
- {
- int ind = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, smps_strs[i]);
- if ( ind == -1 )
- {
- error("Sample '%s' not in input vcf file.\n", smps_strs[i]);
- } else {
- args.selected_smps[ind] = 1;
- }
- free(smps_strs[i]);
- }
- free(smps_strs);
-
- /*
- fprintf(stderr, "Selected samples array:[");
- for (i=0;i<args.nsmp;i++)
- {
- fprintf(stderr, " %i", args.selected_smps[i]);
- }
- fprintf(stderr, " ]\n");
- */
-
- if ( bcf_hdr_id2int(args.hdr, BCF_DT_ID, "GT")<0 ) error("[E::%s] GT not present in the header\n", __func__);
-
- args.gt_arr = NULL;
-
- return 0;
-}
-
-
-/*
- * GT field (genotype) comparison function.
- */
-bcf1_t *process(bcf1_t *rec)
-{
- uint64_t i;
- bcf_unpack(rec, BCF_UN_FMT); // unpack the Format fields, including the GT field
- int gte_smp = 0; // number GT array entries per sample (should be 2, one entry per allele)
- args.ngt_arr = 0; /*! hold the number of current GT array entries */
- if ( (gte_smp = bcf_get_genotypes(args.hdr, rec, &(args.gt_arr), &(args.ngt_arr) ) ) <= 0 )
- {
- error("GT not present at %s: %d\n", args.hdr->id[BCF_DT_CTG][rec->rid].key, rec->pos+1);
- }
-
- gte_smp /= args.nsmp; // divide total number of genotypes array entries (= args.ngt_arr) by number of samples
-
- // initialize with missing genotype
- int a1 = 0;
- int a2 = 0;
-
- // initialize with first selected sample genotype that is not missing
- int gt = -1;
- while ( (a1 == 0) || (a2 == 0) )
- {
- gt++;
- if (gt == args.nsmp) break;
- if (args.selected_smps[gt] == 0) continue;
- a1 = (args.gt_arr + gte_smp * gt)[0];
- if ( gte_smp == 2 ) a2 = (args.gt_arr + gte_smp * gt)[1];
- else if ( gte_smp == 1 ) a2 = bcf_int32_vector_end;
- else error("GTsubset does not support ploidy higher than 2.\n");
- }
-// fprintf(stderr, "a1: %i a2: %i\n", a1, a2);
-
- // check all genotypes if they match (for included samples) or disagree (for samples not included)
- gt = 0;
- for ( i = 0; i < args.nsmp; i++ )
- {
- int *gt_ptr = args.gt_arr + gte_smp * i;
-
- int b1 = gt_ptr[0];
- int b2;
- if ( gte_smp == 2 ) // two entries available per sample, padded with missing values for haploid genotypes
- {
- b2 = gt_ptr[1];
- }
- else if (gte_smp == 1 ) // use vector end value for second entry, if only one is available
- {
- b2 = bcf_int32_vector_end;
- }
- else
- {
- error("GTsubset does not support ploidy higher than 2.\n");
- }
-
- // fprintf(stderr, "b1: %i b2: %i\n", b1, b2);
- /* missing genotypes are counted as always passing, as they neither
- * mismatch the initial selected genotype for a selected sample, nor
- * do they match the initial selected genotype for an excluded sample's
- * genotype */
- if ( (b1 == 0) || (b2 == 0) )
- {
- gt++;
-// fprintf(stderr, "missing => pass\n");
- continue;
- }
- else if ( args.selected_smps[i] == 1 )
- {
- if ( (b1 == a1) && (b2 == a2) )
- {
- gt++;
-// fprintf(stderr, "match => pass\n");
- continue;
- }
- else
- {
-// fprintf(stderr, "no match => fail\n");
- break;
- }
- }
- else if ( args.selected_smps[i] == 0 )
- {
- if ( (b1 != a1 ) || (b2 != a2) )
- {
- gt++;
- // fprintf(stderr, "no match => pass\n");
- continue;
- }
- else
- {
-// fprintf(stderr, "match => fail\n");
- break;
- }
- }
- }
- if ( gt == args.nsmp )
- {
- return rec;
- }
- else
- {
- return NULL;
- }
-}
-
-void destroy(void)
-{
- /* freeing up args */
- bcf_hdr_destroy(args.hdr);
- free(args.gt_arr);
- free(args.selected_smps);
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugins/GTsubset.c -- output only positions where the selected samples exclusively
- share a genotype, i.e. all selected samples must have the same
- genotype (including both alleles) and none of the unselected
- samples can have the same genotype
-
- Copyright (C) 2016 Computational Biology of Infection Research,
- Helmholtz Centre for Infection Research, Braunschweig,
- Germany
-
- Author: David Laehnemann <david.laehnemann@hhu.de>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <unistd.h>
-#include <inttypes.h>
-
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-
-#include "bcftools.h"
-
-typedef struct _args_t
-{
- bcf_hdr_t *hdr; /*! VCF file header */
- int *gt_arr; /*! temporary array, to store GTs of current line/record */
- int ngt_arr; /*! hold the number of current GT array entries */
- int nsmp; /*! number of samples, can be determined from header but is needed in multiple contexts */
- int n_sel_smps; /*! number of selected samples who should exclusively share genotypes */
- int *selected_smps; /*! pointer to start of array containing 1 at indices corresponding to selected samples in header dict and 0 at others*/
-}
-args_t;
-
-static args_t args;
-
-const char *about(void)
-{
- return "Output only sites where the requested samples all exclusively share a genotype (GT).\n";
-}
-
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Output only sites where the requested samples all exclusively share a genotype (GT), i.e.\n"
- " all selected samples must have the same GT, while non of the others can have it.\n"
- "Usage: bcftools +GTsubset <multisample.bcf/.vcf.gz> [General Options] -- [Plugin Options] \n"
- "\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -s,--sample-list comma-separated list of samples; only those sites where all of these\n"
- " samples exclusively share their genotype are given as output\n"
- "\n"
- "Example:\n"
- " bcftools +GTsubset in.vcf -- -s SMP1,SMP2 \n"
- "\n";
-}
-
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- memset(&args,0,sizeof(args_t));
-
- int i;
-
- static struct option loptions[] =
- {
- {"help", no_argument, 0,'h'},
- {"sample-list", required_argument, 0,'s'},
- {0,0,0,0}
- };
-
- char **smps_strs = NULL;
-
- int c;
- while ((c = getopt_long(argc, argv, "?s:h",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 's': smps_strs = hts_readlist(optarg,0,&(args.n_sel_smps));
- if ( args.n_sel_smps == 0 )
- {
- fprintf(bcftools_stderr, "Sample specification not valid.\n");
- error("%s", usage());
- }
- break;
- case 'h': usage(); break;
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( optind != argc ) usage(); // too many files given
-
- args.hdr = bcf_hdr_dup(in);
-
- // Samples parsing from header and input option
- if ( !bcf_hdr_nsamples(args.hdr) )
- {
- error("No samples in input file.\n");
- }
- args.nsmp = bcf_hdr_nsamples(args.hdr);
- args.selected_smps = (int*) calloc(args.nsmp,sizeof(int));
- for ( i = 0; i < args.n_sel_smps; i++ )
- {
- int ind = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, smps_strs[i]);
- if ( ind == -1 )
- {
- error("Sample '%s' not in input vcf file.\n", smps_strs[i]);
- } else {
- args.selected_smps[ind] = 1;
- }
- free(smps_strs[i]);
- }
- free(smps_strs);
-
- /*
- fprintf(bcftools_stderr, "Selected samples array:[");
- for (i=0;i<args.nsmp;i++)
- {
- fprintf(bcftools_stderr, " %i", args.selected_smps[i]);
- }
- fprintf(bcftools_stderr, " ]\n");
- */
-
- if ( bcf_hdr_id2int(args.hdr, BCF_DT_ID, "GT")<0 ) error("[E::%s] GT not present in the header\n", __func__);
-
- args.gt_arr = NULL;
-
- return 0;
-}
-
-
-/*
- * GT field (genotype) comparison function.
- */
-bcf1_t *process(bcf1_t *rec)
-{
- uint64_t i;
- bcf_unpack(rec, BCF_UN_FMT); // unpack the Format fields, including the GT field
- int gte_smp = 0; // number GT array entries per sample (should be 2, one entry per allele)
- args.ngt_arr = 0; /*! hold the number of current GT array entries */
- if ( (gte_smp = bcf_get_genotypes(args.hdr, rec, &(args.gt_arr), &(args.ngt_arr) ) ) <= 0 )
- {
- error("GT not present at %s: %d\n", args.hdr->id[BCF_DT_CTG][rec->rid].key, rec->pos+1);
- }
-
- gte_smp /= args.nsmp; // divide total number of genotypes array entries (= args.ngt_arr) by number of samples
-
- // initialize with missing genotype
- int a1 = 0;
- int a2 = 0;
-
- // initialize with first selected sample genotype that is not missing
- int gt = -1;
- while ( (a1 == 0) || (a2 == 0) )
- {
- gt++;
- if (gt == args.nsmp) break;
- if (args.selected_smps[gt] == 0) continue;
- a1 = (args.gt_arr + gte_smp * gt)[0];
- if ( gte_smp == 2 ) a2 = (args.gt_arr + gte_smp * gt)[1];
- else if ( gte_smp == 1 ) a2 = bcf_int32_vector_end;
- else error("GTsubset does not support ploidy higher than 2.\n");
- }
-// fprintf(bcftools_stderr, "a1: %i a2: %i\n", a1, a2);
-
- // check all genotypes if they match (for included samples) or disagree (for samples not included)
- gt = 0;
- for ( i = 0; i < args.nsmp; i++ )
- {
- int *gt_ptr = args.gt_arr + gte_smp * i;
-
- int b1 = gt_ptr[0];
- int b2;
- if ( gte_smp == 2 ) // two entries available per sample, padded with missing values for haploid genotypes
- {
- b2 = gt_ptr[1];
- }
- else if (gte_smp == 1 ) // use vector end value for second entry, if only one is available
- {
- b2 = bcf_int32_vector_end;
- }
- else
- {
- error("GTsubset does not support ploidy higher than 2.\n");
- }
-
- // fprintf(bcftools_stderr, "b1: %i b2: %i\n", b1, b2);
- /* missing genotypes are counted as always passing, as they neither
- * mismatch the initial selected genotype for a selected sample, nor
- * do they match the initial selected genotype for an excluded sample's
- * genotype */
- if ( (b1 == 0) || (b2 == 0) )
- {
- gt++;
-// fprintf(bcftools_stderr, "missing => pass\n");
- continue;
- }
- else if ( args.selected_smps[i] == 1 )
- {
- if ( (b1 == a1) && (b2 == a2) )
- {
- gt++;
-// fprintf(bcftools_stderr, "match => pass\n");
- continue;
- }
- else
- {
-// fprintf(bcftools_stderr, "no match => fail\n");
- break;
- }
- }
- else if ( args.selected_smps[i] == 0 )
- {
- if ( (b1 != a1 ) || (b2 != a2) )
- {
- gt++;
- // fprintf(bcftools_stderr, "no match => pass\n");
- continue;
- }
- else
- {
-// fprintf(bcftools_stderr, "match => fail\n");
- break;
- }
- }
- }
- if ( gt == args.nsmp )
- {
- return rec;
- }
- else
- {
- return NULL;
- }
-}
-
-void destroy(void)
-{
- /* freeing up args */
- bcf_hdr_destroy(args.hdr);
- free(args.gt_arr);
- free(args.selected_smps);
-}
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/kfunc.h>
-#include <inttypes.h>
-#include "bcftools.h"
-#include "convert.h"
-
-typedef struct
-{
- int smpl,ctrl; // VCF sample index
- const char *smpl_name, *ctrl_name;
-}
-pair_t;
-
-typedef struct
-{
- bcf_hdr_t *hdr;
- pair_t *pair;
- int npair, mpair, min_dp, min_alt_dp;
- int32_t *ad_arr;
- int mad_arr;
- double th;
- convert_t *convert;
- kstring_t str;
- uint64_t nsite,ncmp;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Find positions with wildly varying ALT allele frequency (Fisher test on FMT/AD).\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Find positions with wildly varying ALT allele frequency (Fisher test on FMT/AD).\n"
- "Usage: bcftools +ad-bias [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -a, --min-alt-dp <int> Minimum required alternate allele depth [1]\n"
- " -d, --min-dp <int> Minimum required depth [0]\n"
- " -f, --format <string> Optional tags to append to output (`bcftools query` style of format)\n"
- " -s, --samples <file> List of sample pairs, one tab-delimited pair per line\n"
- " -t, --threshold <float> Output only hits with p-value smaller than <float> [1e-3]\n"
- "\n"
- "Example:\n"
- " bcftools +ad-bias file.bcf -- -t 1e-3 -s samples.txt\n"
- "\n";
-}
-
-void parse_samples(args_t *args, char *fname)
-{
- htsFile *fp = hts_open(fname, "r");
- if ( !fp ) error("Could not read: %s\n", fname);
-
- kstring_t str = {0,0,0};
- if ( hts_getline(fp, KS_SEP_LINE, &str) <= 0 ) error("Empty file: %s\n", fname);
-
- int moff = 0, *off = NULL;
- do
- {
- // HPSI0513i-veqz_6 HPSI0513pf-veqz
- int ncols = ksplit_core(str.s,'\t',&moff,&off);
- if ( ncols<2 ) error("Could not parse the sample file: %s\n", str.s);
-
- int smpl = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[0]]);
- if ( smpl<0 ) continue;
- int ctrl = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[1]]);
- if ( ctrl<0 ) continue;
-
- args->npair++;
- hts_expand0(pair_t,args->npair,args->mpair,args->pair);
- pair_t *pair = &args->pair[args->npair-1];
- pair->ctrl = ctrl;
- pair->smpl = smpl;
- pair->smpl_name = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,pair->smpl);
- pair->ctrl_name = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,pair->ctrl);
- } while ( hts_getline(fp, KS_SEP_LINE, &str)>=0 );
-
- free(str.s);
- free(off);
- hts_close(fp);
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- memset(&args,0,sizeof(args_t));
- args.hdr = in;
- args.th = 1e-3;
- args.min_alt_dp = 1;
- char *fname = NULL, *format = NULL;
- static struct option loptions[] =
- {
- {"min-dp",required_argument,NULL,'d'},
- {"min-alt-dp",required_argument,NULL,'a'},
- {"format",required_argument,NULL,'f'},
- {"samples",required_argument,NULL,'s'},
- {"threshold",required_argument,NULL,'t'},
- {NULL,0,NULL,0}
- };
- int c;
- char *tmp;
- while ((c = getopt_long(argc, argv, "?hs:t:f:d:a:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'a':
- args.min_alt_dp = strtol(optarg,&tmp,10);
- if ( *tmp ) error("Could not parse: -a %s\n", optarg);
- break;
- case 'd':
- args.min_dp = strtol(optarg,&tmp,10);
- if ( *tmp ) error("Could not parse: -d %s\n", optarg);
- break;
- case 't':
- args.th = strtod(optarg,&tmp);
- if ( *tmp ) error("Could not parse: -t %s\n", optarg);
- break;
- case 's': fname = optarg; break;
- case 'f': format = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( !fname ) error("Expected the -s option\n");
- parse_samples(&args, fname);
- if ( format ) args.convert = convert_init(args.hdr, NULL, 0, format);
- printf("# This file was produced by: bcftools +ad-bias(%s+htslib-%s)\n", bcftools_version(),hts_version());
- printf("# The command line was:\tbcftools +ad-bias %s", argv[0]);
- for (c=1; c<argc; c++) printf(" %s",argv[c]);
- printf("\n#\n");
- printf("# FT, Fisher Test\t[2]Sample\t[3]Control\t[4]Chrom\t[5]Pos\t[6]smpl.nREF\t[7]smpl.nALT\t[8]ctrl.nREF\t[9]ctrl.nALT\t[10]P-value");
- if ( format ) printf("\t[11-]User data: %s", format);
- printf("\n");
- return 1;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int nad = bcf_get_format_int32(args.hdr, rec, "AD", &args.ad_arr, &args.mad_arr);
- if ( nad<0 ) return NULL;
- nad /= bcf_hdr_nsamples(args.hdr);
-
- if ( args.convert ) convert_line(args.convert, rec, &args.str);
- args.nsite++;
-
- int i;
- for (i=0; i<args.npair; i++)
- {
- pair_t *pair = &args.pair[i];
- int32_t *aptr = args.ad_arr + nad*pair->smpl;
- int32_t *bptr = args.ad_arr + nad*pair->ctrl;
-
- if ( aptr[0]==bcf_int32_missing ) continue;
- if ( bptr[0]==bcf_int32_missing ) continue;
- if ( aptr[0]+aptr[1] < args.min_dp ) continue;
- if ( bptr[0]+bptr[1] < args.min_dp ) continue;
- if ( aptr[1] < args.min_alt_dp && bptr[1] < args.min_alt_dp ) continue;
-
- args.ncmp++;
-
- int n11 = aptr[0], n12 = aptr[1];
- int n21 = bptr[0], n22 = bptr[1];
- double left, right, fisher;
- kt_fisher_exact(n11,n12,n21,n22, &left,&right,&fisher);
- if ( fisher >= args.th ) continue;
-
- printf("FT\t%s\t%s\t%s\t%d\t%d\t%d\t%d\t%d\t%e",
- pair->smpl_name,pair->ctrl_name,
- bcf_hdr_id2name(args.hdr,rec->rid), rec->pos+1,
- n11,n12,n21,n22, fisher
- );
- if ( args.convert ) printf("\t%s", args.str.s);
- printf("\n");
- }
- return NULL;
-}
-
-void destroy(void)
-{
- printf("# SN, Summary Numbers\t[2]Number of Pairs\t[3]Number of Sites\t[4]Number of comparisons\t[5]P-value output threshold\n");
- printf("SN\t%d\t%"PRId64"\t%"PRId64"\t%e\n",args.npair,args.nsite,args.ncmp,args.th);
- if (args.convert) convert_destroy(args.convert);
- free(args.str.s);
- free(args.pair);
- free(args.ad_arr);
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/kfunc.h>
-#include <inttypes.h>
-#include "bcftools.h"
-#include "convert.h"
-
-typedef struct
-{
- int smpl,ctrl; // VCF sample index
- const char *smpl_name, *ctrl_name;
-}
-pair_t;
-
-typedef struct
-{
- bcf_hdr_t *hdr;
- pair_t *pair;
- int npair, mpair, min_dp, min_alt_dp;
- int32_t *ad_arr;
- int mad_arr;
- double th;
- convert_t *convert;
- kstring_t str;
- uint64_t nsite,ncmp;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Find positions with wildly varying ALT allele frequency (Fisher test on FMT/AD).\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Find positions with wildly varying ALT allele frequency (Fisher test on FMT/AD).\n"
- "Usage: bcftools +ad-bias [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -a, --min-alt-dp <int> Minimum required alternate allele depth [1]\n"
- " -d, --min-dp <int> Minimum required depth [0]\n"
- " -f, --format <string> Optional tags to append to output (`bcftools query` style of format)\n"
- " -s, --samples <file> List of sample pairs, one tab-delimited pair per line\n"
- " -t, --threshold <float> Output only hits with p-value smaller than <float> [1e-3]\n"
- "\n"
- "Example:\n"
- " bcftools +ad-bias file.bcf -- -t 1e-3 -s samples.txt\n"
- "\n";
-}
-
-void parse_samples(args_t *args, char *fname)
-{
- htsFile *fp = hts_open(fname, "r");
- if ( !fp ) error("Could not read: %s\n", fname);
-
- kstring_t str = {0,0,0};
- if ( hts_getline(fp, KS_SEP_LINE, &str) <= 0 ) error("Empty file: %s\n", fname);
-
- int moff = 0, *off = NULL;
- do
- {
- // HPSI0513i-veqz_6 HPSI0513pf-veqz
- int ncols = ksplit_core(str.s,'\t',&moff,&off);
- if ( ncols<2 ) error("Could not parse the sample file: %s\n", str.s);
-
- int smpl = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[0]]);
- if ( smpl<0 ) continue;
- int ctrl = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[1]]);
- if ( ctrl<0 ) continue;
-
- args->npair++;
- hts_expand0(pair_t,args->npair,args->mpair,args->pair);
- pair_t *pair = &args->pair[args->npair-1];
- pair->ctrl = ctrl;
- pair->smpl = smpl;
- pair->smpl_name = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,pair->smpl);
- pair->ctrl_name = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,pair->ctrl);
- } while ( hts_getline(fp, KS_SEP_LINE, &str)>=0 );
-
- free(str.s);
- free(off);
- hts_close(fp);
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- memset(&args,0,sizeof(args_t));
- args.hdr = in;
- args.th = 1e-3;
- args.min_alt_dp = 1;
- char *fname = NULL, *format = NULL;
- static struct option loptions[] =
- {
- {"min-dp",required_argument,NULL,'d'},
- {"min-alt-dp",required_argument,NULL,'a'},
- {"format",required_argument,NULL,'f'},
- {"samples",required_argument,NULL,'s'},
- {"threshold",required_argument,NULL,'t'},
- {NULL,0,NULL,0}
- };
- int c;
- char *tmp;
- while ((c = getopt_long(argc, argv, "?hs:t:f:d:a:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'a':
- args.min_alt_dp = strtol(optarg,&tmp,10);
- if ( *tmp ) error("Could not parse: -a %s\n", optarg);
- break;
- case 'd':
- args.min_dp = strtol(optarg,&tmp,10);
- if ( *tmp ) error("Could not parse: -d %s\n", optarg);
- break;
- case 't':
- args.th = strtod(optarg,&tmp);
- if ( *tmp ) error("Could not parse: -t %s\n", optarg);
- break;
- case 's': fname = optarg; break;
- case 'f': format = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( !fname ) error("Expected the -s option\n");
- parse_samples(&args, fname);
- if ( format ) args.convert = convert_init(args.hdr, NULL, 0, format);
- fprintf(bcftools_stdout, "# This file was produced by: bcftools +ad-bias(%s+htslib-%s)\n", bcftools_version(),hts_version());
- fprintf(bcftools_stdout, "# The command line was:\tbcftools +ad-bias %s", argv[0]);
- for (c=1; c<argc; c++) fprintf(bcftools_stdout, " %s",argv[c]);
- fprintf(bcftools_stdout, "\n#\n");
- fprintf(bcftools_stdout, "# FT, Fisher Test\t[2]Sample\t[3]Control\t[4]Chrom\t[5]Pos\t[6]smpl.nREF\t[7]smpl.nALT\t[8]ctrl.nREF\t[9]ctrl.nALT\t[10]P-value");
- if ( format ) fprintf(bcftools_stdout, "\t[11-]User data: %s", format);
- fprintf(bcftools_stdout, "\n");
- return 1;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int nad = bcf_get_format_int32(args.hdr, rec, "AD", &args.ad_arr, &args.mad_arr);
- if ( nad<0 ) return NULL;
- nad /= bcf_hdr_nsamples(args.hdr);
-
- if ( args.convert ) convert_line(args.convert, rec, &args.str);
- args.nsite++;
-
- int i;
- for (i=0; i<args.npair; i++)
- {
- pair_t *pair = &args.pair[i];
- int32_t *aptr = args.ad_arr + nad*pair->smpl;
- int32_t *bptr = args.ad_arr + nad*pair->ctrl;
-
- if ( aptr[0]==bcf_int32_missing ) continue;
- if ( bptr[0]==bcf_int32_missing ) continue;
- if ( aptr[0]+aptr[1] < args.min_dp ) continue;
- if ( bptr[0]+bptr[1] < args.min_dp ) continue;
- if ( aptr[1] < args.min_alt_dp && bptr[1] < args.min_alt_dp ) continue;
-
- args.ncmp++;
-
- int n11 = aptr[0], n12 = aptr[1];
- int n21 = bptr[0], n22 = bptr[1];
- double left, right, fisher;
- kt_fisher_exact(n11,n12,n21,n22, &left,&right,&fisher);
- if ( fisher >= args.th ) continue;
-
- fprintf(bcftools_stdout, "FT\t%s\t%s\t%s\t%d\t%d\t%d\t%d\t%d\t%e",
- pair->smpl_name,pair->ctrl_name,
- bcf_hdr_id2name(args.hdr,rec->rid), rec->pos+1,
- n11,n12,n21,n22, fisher
- );
- if ( args.convert ) fprintf(bcftools_stdout, "\t%s", args.str.s);
- fprintf(bcftools_stdout, "\n");
- }
- return NULL;
-}
-
-void destroy(void)
-{
- fprintf(bcftools_stdout, "# SN, Summary Numbers\t[2]Number of Pairs\t[3]Number of Sites\t[4]Number of comparisons\t[5]P-value output threshold\n");
- fprintf(bcftools_stdout, "SN\t%d\t%"PRId64"\t%"PRId64"\t%e\n",args.npair,args.nsite,args.ncmp,args.th);
- if (args.convert) convert_destroy(args.convert);
- free(args.str.s);
- free(args.pair);
- free(args.ad_arr);
-}
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <inttypes.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include "bcftools.h"
-#include "bin.h"
-
-typedef struct
-{
- char *af_tag;
- bcf_hdr_t *hdr;
- int32_t *gt, ngt, naf;
- float *af, list_min, list_max;
- bin_t *dev_bins, *prob_bins;
- uint64_t *dev_dist, *prob_dist;
-}
-args_t;
-
-args_t *args;
-
-const char *about(void)
-{
- return "AF and GT probability distribution stats.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Collect AF deviation stats and GT probability distribution\n"
- " given AF and assuming HWE\n"
- "Usage: bcftools +af-dist [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -d, --dev-bins <list> AF deviation bins\n"
- " -l, --list <min,max> list genotypes from the given bin (for debugging)\n"
- " -p, --prob-bins <list> probability distribution bins\n"
- " -t, --af-tag <tag> VCF INFO tag to use [AF]\n"
- "\n"
- "Default binning:\n"
- " -d: 0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1\n"
- " -p: 0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1\n"
- "Example:\n"
- " bcftools +af-tag file.bcf -- -t EUR_AF -p bins.txt\n"
- "\n";
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- args = (args_t*) calloc(1,sizeof(args_t));
- char *dev_bins = "0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1";
- char *prob_bins = "0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1";
- args->hdr = in;
- args->af_tag = "AF";
- args->list_min = -1;
- static struct option loptions[] =
- {
- {"list",required_argument,NULL,'l'},
- {"dev-bins",required_argument,NULL,'d'},
- {"prob-bins",required_argument,NULL,'p'},
- {"af-tag",required_argument,NULL,'t'},
- {NULL,0,NULL,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?ht:d:p:l:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'l':
- {
- char *a,*b;
- args->list_min = strtod(optarg,&a);
- if ( a==optarg || *a!=',' ) error("Could not parse: --list %s\n", optarg);
- args->list_max = strtod(a+1,&b);
- if ( a+1==b || *b ) error("Could not parse: --list %s\n", optarg);
- break;
- }
- case 'd': dev_bins = optarg; break;
- case 'p': prob_bins = optarg; break;
- case 't': args->af_tag = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
-
- args->dev_bins = bin_init(dev_bins,0,1);
- int nbins = bin_get_size(args->dev_bins);
- args->dev_dist = (uint64_t*)calloc(nbins,sizeof(*args->dev_dist));
-
- args->prob_bins = bin_init(prob_bins,0,1);
- nbins = bin_get_size(args->prob_bins);
- args->prob_dist = (uint64_t*)calloc(nbins,sizeof(*args->prob_dist));
-
- printf("# This file was produced by: bcftools +af-dist(%s+htslib-%s)\n", bcftools_version(),hts_version());
- printf("# The command line was:\tbcftools +af-dist %s", argv[0]);
- for (c=1; c<argc; c++) printf(" %s",argv[c]);
- printf("\n#\n");
-
- if ( args->list_min!=-1 )
- printf("# GT, genotypes with P(AF) in [%f,%f]; [2]Chromosome\t[3]Position[4]Sample\t[5]Genotype\t[6]AF-based probability\n",args->list_min,args->list_max);
-
- return 1;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int naf = bcf_get_info_float(args->hdr,rec,args->af_tag,&args->af,&args->naf);
- if ( naf<=0 ) return NULL;
- float af = args->af[0];
-
- float pRA = 2*af*(1-af);
- float pAA = af*af;
- int iRA = bin_get_idx(args->prob_bins,pRA);
- int iAA = bin_get_idx(args->prob_bins,pAA);
-
- int list_RA = args->list_min==-1 || pRA < args->list_min || pRA > args->list_max ? 0 : 1;
- int list_AA = args->list_min==-1 || pAA < args->list_min || pAA > args->list_max ? 0 : 1;
- const char *chr = bcf_seqname(args->hdr,rec);
-
- int ngt = bcf_get_genotypes(args->hdr, rec, &args->gt, &args->ngt);
- int i, j, nsmpl = bcf_hdr_nsamples(args->hdr);
- int nals = 0, nalt = 0;
- ngt /= nsmpl;
- for (i=0; i<nsmpl; i++)
- {
- int32_t *ptr = args->gt + i*ngt;
- int dosage = 0;
- for (j=0; j<ngt; j++)
- {
- if ( bcf_gt_is_missing(ptr[j]) ) break;
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( bcf_gt_allele(ptr[j])==1 ) dosage++;
- }
- if ( j!=ngt ) continue;
-
- nals += j;
- nalt += dosage;
-
- if ( dosage==1 )
- {
- args->prob_dist[iRA]++;
- if ( list_RA ) printf("GT\t%s\t%d\t%s\t1\t%f\n",chr,rec->pos+1,args->hdr->samples[i],pRA);
- }
- else if ( dosage==2 )
- {
- args->prob_dist[iAA]++;
- if ( list_AA ) printf("GT\t%s\t%d\t%s\t2\t%f\n",chr,rec->pos+1,args->hdr->samples[i],pAA);
- }
- }
-
- if ( nals && (nalt || af) )
- {
- float af_dev = fabs(af - (float)nalt/nals);
- int iAF = bin_get_idx(args->dev_bins,af_dev);
- args->dev_dist[iAF]++;
- }
-
- return NULL;
-}
-
-void destroy(void)
-{
- printf("# PROB_DIST, genotype probability distribution, assumes HWE\n");
- int i, n;
- n = bin_get_size(args->prob_bins);
- for (i=0; i<n-1; i++)
- {
- float min = bin_get_value(args->prob_bins,i);
- float max = bin_get_value(args->prob_bins,i+1);
- printf("PROB_DIST\t%f\t%f\t%"PRId64"\n", min,max,args->prob_dist[i]);
- }
- printf("# DEV_DIST, distribution of AF deviation, based on %s and INFO/AN, AC calculated on the fly\n", args->af_tag);
- n = bin_get_size(args->dev_bins);
- for (i=0; i<n-1; i++)
- {
- float min = bin_get_value(args->dev_bins,i);
- float max = bin_get_value(args->dev_bins,i+1);
- printf("DEV_DIST\t%f\t%f\t%"PRId64"\n", min,max,args->dev_dist[i]);
- }
- bin_destroy(args->dev_bins);
- bin_destroy(args->prob_bins);
- free(args->dev_dist);
- free(args->prob_dist);
- free(args->gt);
- free(args->af);
- free(args);
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <inttypes.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include "bcftools.h"
-#include "bin.h"
-
-typedef struct
-{
- char *af_tag;
- bcf_hdr_t *hdr;
- int32_t *gt, ngt, naf;
- float *af, list_min, list_max;
- bin_t *dev_bins, *prob_bins;
- uint64_t *dev_dist, *prob_dist;
-}
-args_t;
-
-args_t *args;
-
-const char *about(void)
-{
- return "AF and GT probability distribution stats.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Collect AF deviation stats and GT probability distribution\n"
- " given AF and assuming HWE\n"
- "Usage: bcftools +af-dist [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -d, --dev-bins <list> AF deviation bins\n"
- " -l, --list <min,max> list genotypes from the given bin (for debugging)\n"
- " -p, --prob-bins <list> probability distribution bins\n"
- " -t, --af-tag <tag> VCF INFO tag to use [AF]\n"
- "\n"
- "Default binning:\n"
- " -d: 0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1\n"
- " -p: 0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1\n"
- "Example:\n"
- " bcftools +af-tag file.bcf -- -t EUR_AF -p bins.txt\n"
- "\n";
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- args = (args_t*) calloc(1,sizeof(args_t));
- char *dev_bins = "0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1";
- char *prob_bins = "0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1";
- args->hdr = in;
- args->af_tag = "AF";
- args->list_min = -1;
- static struct option loptions[] =
- {
- {"list",required_argument,NULL,'l'},
- {"dev-bins",required_argument,NULL,'d'},
- {"prob-bins",required_argument,NULL,'p'},
- {"af-tag",required_argument,NULL,'t'},
- {NULL,0,NULL,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?ht:d:p:l:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'l':
- {
- char *a,*b;
- args->list_min = strtod(optarg,&a);
- if ( a==optarg || *a!=',' ) error("Could not parse: --list %s\n", optarg);
- args->list_max = strtod(a+1,&b);
- if ( a+1==b || *b ) error("Could not parse: --list %s\n", optarg);
- break;
- }
- case 'd': dev_bins = optarg; break;
- case 'p': prob_bins = optarg; break;
- case 't': args->af_tag = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
-
- args->dev_bins = bin_init(dev_bins,0,1);
- int nbins = bin_get_size(args->dev_bins);
- args->dev_dist = (uint64_t*)calloc(nbins,sizeof(*args->dev_dist));
-
- args->prob_bins = bin_init(prob_bins,0,1);
- nbins = bin_get_size(args->prob_bins);
- args->prob_dist = (uint64_t*)calloc(nbins,sizeof(*args->prob_dist));
-
- fprintf(bcftools_stdout, "# This file was produced by: bcftools +af-dist(%s+htslib-%s)\n", bcftools_version(),hts_version());
- fprintf(bcftools_stdout, "# The command line was:\tbcftools +af-dist %s", argv[0]);
- for (c=1; c<argc; c++) fprintf(bcftools_stdout, " %s",argv[c]);
- fprintf(bcftools_stdout, "\n#\n");
-
- if ( args->list_min!=-1 )
- fprintf(bcftools_stdout, "# GT, genotypes with P(AF) in [%f,%f]; [2]Chromosome\t[3]Position[4]Sample\t[5]Genotype\t[6]AF-based probability\n",args->list_min,args->list_max);
-
- return 1;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int naf = bcf_get_info_float(args->hdr,rec,args->af_tag,&args->af,&args->naf);
- if ( naf<=0 ) return NULL;
- float af = args->af[0];
-
- float pRA = 2*af*(1-af);
- float pAA = af*af;
- int iRA = bin_get_idx(args->prob_bins,pRA);
- int iAA = bin_get_idx(args->prob_bins,pAA);
-
- int list_RA = args->list_min==-1 || pRA < args->list_min || pRA > args->list_max ? 0 : 1;
- int list_AA = args->list_min==-1 || pAA < args->list_min || pAA > args->list_max ? 0 : 1;
- const char *chr = bcf_seqname(args->hdr,rec);
-
- int ngt = bcf_get_genotypes(args->hdr, rec, &args->gt, &args->ngt);
- int i, j, nsmpl = bcf_hdr_nsamples(args->hdr);
- int nals = 0, nalt = 0;
- ngt /= nsmpl;
- for (i=0; i<nsmpl; i++)
- {
- int32_t *ptr = args->gt + i*ngt;
- int dosage = 0;
- for (j=0; j<ngt; j++)
- {
- if ( bcf_gt_is_missing(ptr[j]) ) break;
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( bcf_gt_allele(ptr[j])==1 ) dosage++;
- }
- if ( j!=ngt ) continue;
-
- nals += j;
- nalt += dosage;
-
- if ( dosage==1 )
- {
- args->prob_dist[iRA]++;
- if ( list_RA ) fprintf(bcftools_stdout, "GT\t%s\t%d\t%s\t1\t%f\n",chr,rec->pos+1,args->hdr->samples[i],pRA);
- }
- else if ( dosage==2 )
- {
- args->prob_dist[iAA]++;
- if ( list_AA ) fprintf(bcftools_stdout, "GT\t%s\t%d\t%s\t2\t%f\n",chr,rec->pos+1,args->hdr->samples[i],pAA);
- }
- }
-
- if ( nals && (nalt || af) )
- {
- float af_dev = fabs(af - (float)nalt/nals);
- int iAF = bin_get_idx(args->dev_bins,af_dev);
- args->dev_dist[iAF]++;
- }
-
- return NULL;
-}
-
-void destroy(void)
-{
- fprintf(bcftools_stdout, "# PROB_DIST, genotype probability distribution, assumes HWE\n");
- int i, n;
- n = bin_get_size(args->prob_bins);
- for (i=0; i<n-1; i++)
- {
- float min = bin_get_value(args->prob_bins,i);
- float max = bin_get_value(args->prob_bins,i+1);
- fprintf(bcftools_stdout, "PROB_DIST\t%f\t%f\t%"PRId64"\n", min,max,args->prob_dist[i]);
- }
- fprintf(bcftools_stdout, "# DEV_DIST, distribution of AF deviation, based on %s and INFO/AN, AC calculated on the fly\n", args->af_tag);
- n = bin_get_size(args->dev_bins);
- for (i=0; i<n-1; i++)
- {
- float min = bin_get_value(args->dev_bins,i);
- float max = bin_get_value(args->dev_bins,i+1);
- fprintf(bcftools_stdout, "DEV_DIST\t%f\t%f\t%"PRId64"\n", min,max,args->dev_dist[i]);
- }
- bin_destroy(args->dev_bins);
- bin_destroy(args->prob_bins);
- free(args->dev_dist);
- free(args->prob_dist);
- free(args->gt);
- free(args->af);
- free(args);
-}
-
-
+++ /dev/null
-/*
- Copyright (C) 2017 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <stdint.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include <htslib/kseq.h>
-#include <inttypes.h>
-#include <unistd.h>
-#include "bcftools.h"
-
-typedef struct
-{
- char *sample;
- int beg,end,ploidy;
-}
-dat_t;
-
-typedef struct
-{
- int argc;
- char **argv;
- int rid, gt_id, ndat;
- dat_t *dat;
- bcf_hdr_t *hdr;
-}
-args_t;
-
-static args_t *args;
-
-const char *about(void)
-{
- return "Check if ploidy of samples is consistent for all sites\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Check if ploidy of samples is consistent for all sites.\n"
- "Usage: bcftools +check-ploidy [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Example:\n"
- " bcftools +check-ploidy file.bcf\n"
- "\n";
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->hdr = in;
- args->ndat = bcf_hdr_nsamples(args->hdr);
- args->dat = (dat_t*) calloc(args->ndat,sizeof(dat_t));
- int i;
- for (i=0; i<args->ndat; i++) args->dat[i].sample = args->hdr->samples[i];
- args->rid = -1;
- args->gt_id = bcf_hdr_id2int(args->hdr,BCF_DT_ID,"GT");
- if ( args->gt_id<0 ) error("Error: GT field is not present\n");
- printf("# [1]Sample\t[2]Chromosome\t[3]Region Start\t[4]Region End\t[5]Ploidy\n");
- return 1;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int i;
-
- bcf_unpack(rec, BCF_UN_FMT);
- bcf_fmt_t *fmt_gt = NULL;
- for (i=0; i<rec->n_fmt; i++)
- if ( rec->d.fmt[i].id==args->gt_id ) { fmt_gt = &rec->d.fmt[i]; break; }
- if ( !fmt_gt ) return NULL; // no GT tag
-
- if ( args->ndat != rec->n_sample )
- error("Incorrect number of samples at %s:%d .. found %d, expected %d\n",bcf_seqname(args->hdr,rec),rec->pos+1,rec->n_sample,args->ndat);
-
- if ( args->rid!=rec->rid && args->rid!=-1 )
- {
- for (i=0; i<args->ndat; i++)
- {
- dat_t *dat = &args->dat[i];
- if ( dat->ploidy!=0 ) printf("%s\t%s\t%d\t%d\t%d\n", dat->sample,bcf_seqname(args->hdr,rec),dat->beg+1,dat->end+1,dat->ploidy);
- dat->ploidy = 0;
- }
- }
- args->rid = rec->rid;
-
- #define BRANCH_INT(type_t,vector_end) \
- { \
- for (i=0; i<rec->n_sample; i++) \
- { \
- type_t *p = (type_t*) (fmt_gt->p + i*fmt_gt->size); \
- int nal, missing = 0; \
- for (nal=0; nal<fmt_gt->n; nal++) \
- { \
- if ( p[nal]==vector_end ) break; /* smaller ploidy */ \
- if ( bcf_gt_is_missing(p[nal]) ) { missing=1; break; } /* missing allele */ \
- } \
- if ( !nal || missing ) continue; /* missing genotype */ \
- dat_t *dat = &args->dat[i]; \
- if ( dat->ploidy==nal ) \
- { \
- dat->end = rec->pos; \
- continue; \
- } \
- if ( dat->ploidy!=0 ) \
- printf("%s\t%s\t%d\t%d\t%d\n", dat->sample,bcf_seqname(args->hdr,rec),dat->beg+1,dat->end+1,dat->ploidy); \
- dat->ploidy = nal; \
- dat->beg = rec->pos; \
- dat->end = rec->pos; \
- } \
- }
- switch (fmt_gt->type) {
- case BCF_BT_INT8: BRANCH_INT(int8_t, bcf_int8_vector_end); break;
- case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_vector_end); break;
- case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_vector_end); break;
- default: error("The GT type is not recognised: %d at %s:%d\n",fmt_gt->type, bcf_seqname(args->hdr,rec),rec->pos+1); break;
- }
- #undef BRANCH_INT
-
- return NULL;
-}
-
-void destroy(void)
-{
- int i;
- for (i=0; i<args->ndat; i++)
- {
- dat_t *dat = &args->dat[i];
- if ( dat->ploidy!=0 ) printf("%s\t%s\t%d\t%d\t%d\n", dat->sample,bcf_hdr_id2name(args->hdr,args->rid),dat->beg+1,dat->end+1,dat->ploidy);
- dat->ploidy = 0;
- }
- free(args->dat);
- free(args);
-}
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/*
- Copyright (C) 2017 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <stdint.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include <htslib/kseq.h>
-#include <inttypes.h>
-#include <unistd.h>
-#include "bcftools.h"
-
-typedef struct
-{
- char *sample;
- int beg,end,ploidy;
-}
-dat_t;
-
-typedef struct
-{
- int argc;
- char **argv;
- int rid, gt_id, ndat;
- dat_t *dat;
- bcf_hdr_t *hdr;
-}
-args_t;
-
-static args_t *args;
-
-const char *about(void)
-{
- return "Check if ploidy of samples is consistent for all sites\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Check if ploidy of samples is consistent for all sites.\n"
- "Usage: bcftools +check-ploidy [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Example:\n"
- " bcftools +check-ploidy file.bcf\n"
- "\n";
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->hdr = in;
- args->ndat = bcf_hdr_nsamples(args->hdr);
- args->dat = (dat_t*) calloc(args->ndat,sizeof(dat_t));
- int i;
- for (i=0; i<args->ndat; i++) args->dat[i].sample = args->hdr->samples[i];
- args->rid = -1;
- args->gt_id = bcf_hdr_id2int(args->hdr,BCF_DT_ID,"GT");
- if ( args->gt_id<0 ) error("Error: GT field is not present\n");
- fprintf(bcftools_stdout, "# [1]Sample\t[2]Chromosome\t[3]Region Start\t[4]Region End\t[5]Ploidy\n");
- return 1;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int i;
-
- bcf_unpack(rec, BCF_UN_FMT);
- bcf_fmt_t *fmt_gt = NULL;
- for (i=0; i<rec->n_fmt; i++)
- if ( rec->d.fmt[i].id==args->gt_id ) { fmt_gt = &rec->d.fmt[i]; break; }
- if ( !fmt_gt ) return NULL; // no GT tag
-
- if ( args->ndat != rec->n_sample )
- error("Incorrect number of samples at %s:%d .. found %d, expected %d\n",bcf_seqname(args->hdr,rec),rec->pos+1,rec->n_sample,args->ndat);
-
- if ( args->rid!=rec->rid && args->rid!=-1 )
- {
- for (i=0; i<args->ndat; i++)
- {
- dat_t *dat = &args->dat[i];
- if ( dat->ploidy!=0 ) fprintf(bcftools_stdout, "%s\t%s\t%d\t%d\t%d\n", dat->sample,bcf_seqname(args->hdr,rec),dat->beg+1,dat->end+1,dat->ploidy);
- dat->ploidy = 0;
- }
- }
- args->rid = rec->rid;
-
- #define BRANCH_INT(type_t,vector_end) \
- { \
- for (i=0; i<rec->n_sample; i++) \
- { \
- type_t *p = (type_t*) (fmt_gt->p + i*fmt_gt->size); \
- int nal, missing = 0; \
- for (nal=0; nal<fmt_gt->n; nal++) \
- { \
- if ( p[nal]==vector_end ) break; /* smaller ploidy */ \
- if ( bcf_gt_is_missing(p[nal]) ) { missing=1; break; } /* missing allele */ \
- } \
- if ( !nal || missing ) continue; /* missing genotype */ \
- dat_t *dat = &args->dat[i]; \
- if ( dat->ploidy==nal ) \
- { \
- dat->end = rec->pos; \
- continue; \
- } \
- if ( dat->ploidy!=0 ) \
- fprintf(bcftools_stdout, "%s\t%s\t%d\t%d\t%d\n", dat->sample,bcf_seqname(args->hdr,rec),dat->beg+1,dat->end+1,dat->ploidy); \
- dat->ploidy = nal; \
- dat->beg = rec->pos; \
- dat->end = rec->pos; \
- } \
- }
- switch (fmt_gt->type) {
- case BCF_BT_INT8: BRANCH_INT(int8_t, bcf_int8_vector_end); break;
- case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_vector_end); break;
- case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_vector_end); break;
- default: error("The GT type is not recognised: %d at %s:%d\n",fmt_gt->type, bcf_seqname(args->hdr,rec),rec->pos+1); break;
- }
- #undef BRANCH_INT
-
- return NULL;
-}
-
-void destroy(void)
-{
- int i;
- for (i=0; i<args->ndat; i++)
- {
- dat_t *dat = &args->dat[i];
- if ( dat->ploidy!=0 ) fprintf(bcftools_stdout, "%s\t%s\t%d\t%d\t%d\n", dat->sample,bcf_hdr_id2name(args->hdr,args->rid),dat->beg+1,dat->end+1,dat->ploidy);
- dat->ploidy = 0;
- }
- free(args->dat);
- free(args);
-}
-
+++ /dev/null
-/*
- Copyright (C) 2017 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <stdint.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include <htslib/kseq.h>
-#include <inttypes.h>
-#include <unistd.h>
-#include "bcftools.h"
-
-typedef struct
-{
- int argc;
- char **argv, *fname, *region, **regs;
- int region_is_file, nregs, regs_free;
- int *smpl, nsmpl, *nsites, min_sites, gt_id;
- kstring_t tmps;
- bcf1_t *rec;
- tbx_t *tbx;
- hts_idx_t *idx;
- hts_itr_t *itr;
- htsFile *fp;
- bcf_hdr_t *hdr;
-}
-args_t;
-
-const char *about(void)
-{
- return "Print samples without genotypes in a region or chromosome\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Print samples without genotypess in a region (-r/-R) or chromosome (the default)\n"
- "\n"
- "Usage: bcftools +check-sparsity <file.vcf.gz> [Plugin Options]\n"
- "Plugin options:\n"
- " -n, --n-markers <int> minimum number of required markers [1]\n"
- " -r, --regions <chr:beg-end> restrict to comma-separated list of regions\n"
- " -R, --regions-file <file> restrict to regions listed in a file\n"
- "\n";
-}
-
-static void init_data(args_t *args)
-{
- args->fp = hts_open(args->fname,"r");
- if ( !args->fp ) error("Could not read %s\n", args->fname);
- args->hdr = bcf_hdr_read(args->fp);
- if ( !args->hdr ) error("Could not read the header: %s\n", args->fname);
-
- args->rec = bcf_init1();
- args->gt_id = bcf_hdr_id2int(args->hdr,BCF_DT_ID,"GT");
- if ( args->gt_id<0 ) error("Error: GT field is not present\n");
-
- int i;
- args->nsmpl = bcf_hdr_nsamples(args->hdr);
- args->nsites = (int*) calloc(args->nsmpl, sizeof(int));
- args->smpl = (int*) malloc(sizeof(int)*args->nsmpl);
- for (i=0; i<args->nsmpl; i++) args->smpl[i] = i;
-
- if ( strcmp("-",args->fname) ) // not reading from stdin
- {
- if ( hts_get_format(args->fp)->format==vcf )
- {
- args->tbx = tbx_index_load(args->fname);
- if ( !args->tbx && args->region ) error("Could not load the VCF index, please drop the -r/-R option\n");
- }
- else if ( hts_get_format(args->fp)->format==bcf )
- {
- args->idx = bcf_index_load(args->fname);
- if ( !args->idx && args->region ) error("Could not load the BCF index, please drop the -r/-R option\n");
- }
- }
- else if ( args->region ) error("Cannot use index with this file, please drop the -r/-R option\n");
-
- if ( args->tbx || args->idx )
- {
- if ( args->region )
- {
- args->regs = hts_readlist(args->region, args->region_is_file, &args->nregs);
- if ( !args->regs ) error("Could not parse regions: %s\n", args->region);
- args->regs_free = 1;
- }
- else
- args->regs = (char**) (args->tbx ? tbx_seqnames(args->tbx, &args->nregs) : bcf_index_seqnames(args->idx, args->hdr, &args->nregs));
- }
-}
-static void destroy_data(args_t *args)
-{
- int i;
- if ( args->regs_free )
- for (i=0; i<args->nregs; i++) free(args->regs[i]);
- free(args->regs);
- bcf_hdr_destroy(args->hdr);
- bcf_destroy(args->rec);
- free(args->tmps.s);
- free(args->smpl);
- free(args->nsites);
- if ( args->itr ) hts_itr_destroy(args->itr);
- if ( args->tbx ) tbx_destroy(args->tbx);
- if ( args->idx ) hts_idx_destroy(args->idx);
- hts_close(args->fp);
-}
-
-static void report(args_t *args, const char *reg)
-{
- int i;
- for (i=0; i<args->nsmpl; i++)
- printf("%s\t%s\n", reg, args->hdr->samples[args->smpl[i]]);
- args->nsmpl = bcf_hdr_nsamples(args->hdr);
- for (i=0; i<args->nsmpl; i++) args->smpl[i] = i;
- memset(args->nsites, 0, sizeof(int)*args->nsmpl);
-}
-static void test_region(args_t *args, char *reg)
-{
- if ( args->tbx )
- {
- args->itr = tbx_itr_querys(args->tbx,reg);
- if ( !args->itr ) return;
- }
- else if ( args->idx )
- {
- args->itr = bcf_itr_querys(args->idx,args->hdr,reg);
- if ( !args->itr ) return;
- }
-
- int ret,i, rid = -1, nread = 0;
- while (1)
- {
- if ( args->tbx )
- {
- if ( (ret=tbx_itr_next(args->fp, args->tbx, args->itr, &args->tmps)) < 0 ) break; // no more lines
- ret = vcf_parse1(&args->tmps, args->hdr, args->rec);
- if ( ret<0 ) error("Could not parse the line: %s\n", args->tmps.s);
- }
- else if ( args->idx )
- {
- ret = bcf_itr_next(args->fp, args->itr, args->rec);
- if ( ret < -1 ) error("Could not parse a line from %s\n", reg);
- if ( ret < 0 ) break; // no more lines or an error
- }
- else
- {
- if ( args->fp->format.format==vcf )
- {
- if ( (ret=hts_getline(args->fp, KS_SEP_LINE, &args->tmps)) < 0 ) break; // no more lines
- ret = vcf_parse1(&args->tmps, args->hdr, args->rec);
- if ( ret<0 ) error("Could not parse the line: %s\n", args->tmps.s);
- }
- else if ( args->fp->format.format==bcf )
- {
- ret = bcf_read1(args->fp, args->hdr, args->rec);
- if ( ret < -1 ) error("Could not parse %s\n", args->fname);
- if ( ret < 0 ) break; // no more lines or an error
- }
- if ( rid!=-1 && rid!=args->rec->rid )
- {
- report(args, bcf_hdr_id2name(args->hdr,rid));
- nread = 0;
- }
- rid = args->rec->rid;
- }
-
- bcf_unpack(args->rec, BCF_UN_FMT);
- bcf_fmt_t *fmt_gt = NULL;
- for (i=0; i<args->rec->n_fmt; i++)
- if ( args->rec->d.fmt[i].id==args->gt_id ) { fmt_gt = &args->rec->d.fmt[i]; break; }
- if ( !fmt_gt ) continue; // no GT tag
- if ( fmt_gt->n==0 ) continue; // empty?!
- if ( fmt_gt->type!=BCF_BT_INT8 ) error("TODO: the GT fmt_type is not int8!\n");
-
- // update the array of missing samples
- for (i=0; i<args->nsmpl; i++)
- {
- int8_t *ptr = (int8_t*) (fmt_gt->p + args->smpl[i]*fmt_gt->size);
- int ial = 0;
- for (ial=0; ial<fmt_gt->n; ial++)
- if ( ptr[ial]==bcf_gt_missing || ptr[ial]==bcf_int8_vector_end ) break;
- if ( ial==0 ) continue; // missing
- if ( ++args->nsites[i] < args->min_sites ) continue;
- if ( i+1<args->nsmpl )
- {
- memmove(args->smpl+i, args->smpl+i+1, sizeof(int)*(args->nsmpl-i-1));
- memmove(args->nsites+i, args->nsites+i+1, sizeof(int)*(args->nsmpl-i-1));
- }
- args->nsmpl--;
- i--;
- }
- nread = 1;
- if ( !args->nsmpl ) break;
- }
- if ( nread ) report(args, rid==-1 ? reg : bcf_hdr_id2name(args->hdr,rid));
-
- tbx_itr_destroy(args->itr);
- args->itr = NULL;
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->min_sites = 1;
- static struct option loptions[] =
- {
- {"n-markers",required_argument,NULL,'n'},
- {"regions",required_argument,NULL,'r'},
- {"regions-file",required_argument,NULL,'R'},
- {NULL,0,NULL,0}
- };
- int c,i;
- char *tmp;
- while ((c = getopt_long(argc, argv, "vr:R:n:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'n':
- args->min_sites = strtol(optarg,&tmp,10);
- if ( *tmp ) error("Could not parse: -n %s\n", optarg);
- break;
- case 'R': args->region_is_file = 1;
- case 'r': args->region = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
-
- if ( optind>=argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else error("%s",usage_text());
- }
- else args->fname = argv[optind];
- init_data(args);
-
- for (i=0; i<args->nregs; i++) test_region(args, args->regs[i]);
- if ( !args->nregs ) test_region(args, NULL);
-
- destroy_data(args);
- free(args);
- return 0;
-}
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/*
- Copyright (C) 2017 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <stdint.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include <htslib/kseq.h>
-#include <inttypes.h>
-#include <unistd.h>
-#include "bcftools.h"
-
-typedef struct
-{
- int argc;
- char **argv, *fname, *region, **regs;
- int region_is_file, nregs, regs_free;
- int *smpl, nsmpl, *nsites, min_sites, gt_id;
- kstring_t tmps;
- bcf1_t *rec;
- tbx_t *tbx;
- hts_idx_t *idx;
- hts_itr_t *itr;
- htsFile *fp;
- bcf_hdr_t *hdr;
-}
-args_t;
-
-const char *about(void)
-{
- return "Print samples without genotypes in a region or chromosome\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Print samples without genotypess in a region (-r/-R) or chromosome (the default)\n"
- "\n"
- "Usage: bcftools +check-sparsity <file.vcf.gz> [Plugin Options]\n"
- "Plugin options:\n"
- " -n, --n-markers <int> minimum number of required markers [1]\n"
- " -r, --regions <chr:beg-end> restrict to comma-separated list of regions\n"
- " -R, --regions-file <file> restrict to regions listed in a file\n"
- "\n";
-}
-
-static void init_data(args_t *args)
-{
- args->fp = hts_open(args->fname,"r");
- if ( !args->fp ) error("Could not read %s\n", args->fname);
- args->hdr = bcf_hdr_read(args->fp);
- if ( !args->hdr ) error("Could not read the header: %s\n", args->fname);
-
- args->rec = bcf_init1();
- args->gt_id = bcf_hdr_id2int(args->hdr,BCF_DT_ID,"GT");
- if ( args->gt_id<0 ) error("Error: GT field is not present\n");
-
- int i;
- args->nsmpl = bcf_hdr_nsamples(args->hdr);
- args->nsites = (int*) calloc(args->nsmpl, sizeof(int));
- args->smpl = (int*) malloc(sizeof(int)*args->nsmpl);
- for (i=0; i<args->nsmpl; i++) args->smpl[i] = i;
-
- if ( strcmp("-",args->fname) ) // not reading from stdin
- {
- if ( hts_get_format(args->fp)->format==vcf )
- {
- args->tbx = tbx_index_load(args->fname);
- if ( !args->tbx && args->region ) error("Could not load the VCF index, please drop the -r/-R option\n");
- }
- else if ( hts_get_format(args->fp)->format==bcf )
- {
- args->idx = bcf_index_load(args->fname);
- if ( !args->idx && args->region ) error("Could not load the BCF index, please drop the -r/-R option\n");
- }
- }
- else if ( args->region ) error("Cannot use index with this file, please drop the -r/-R option\n");
-
- if ( args->tbx || args->idx )
- {
- if ( args->region )
- {
- args->regs = hts_readlist(args->region, args->region_is_file, &args->nregs);
- if ( !args->regs ) error("Could not parse regions: %s\n", args->region);
- args->regs_free = 1;
- }
- else
- args->regs = (char**) (args->tbx ? tbx_seqnames(args->tbx, &args->nregs) : bcf_index_seqnames(args->idx, args->hdr, &args->nregs));
- }
-}
-static void destroy_data(args_t *args)
-{
- int i;
- if ( args->regs_free )
- for (i=0; i<args->nregs; i++) free(args->regs[i]);
- free(args->regs);
- bcf_hdr_destroy(args->hdr);
- bcf_destroy(args->rec);
- free(args->tmps.s);
- free(args->smpl);
- free(args->nsites);
- if ( args->itr ) hts_itr_destroy(args->itr);
- if ( args->tbx ) tbx_destroy(args->tbx);
- if ( args->idx ) hts_idx_destroy(args->idx);
- hts_close(args->fp);
-}
-
-static void report(args_t *args, const char *reg)
-{
- int i;
- for (i=0; i<args->nsmpl; i++)
- fprintf(bcftools_stdout, "%s\t%s\n", reg, args->hdr->samples[args->smpl[i]]);
- args->nsmpl = bcf_hdr_nsamples(args->hdr);
- for (i=0; i<args->nsmpl; i++) args->smpl[i] = i;
- memset(args->nsites, 0, sizeof(int)*args->nsmpl);
-}
-static void test_region(args_t *args, char *reg)
-{
- if ( args->tbx )
- {
- args->itr = tbx_itr_querys(args->tbx,reg);
- if ( !args->itr ) return;
- }
- else if ( args->idx )
- {
- args->itr = bcf_itr_querys(args->idx,args->hdr,reg);
- if ( !args->itr ) return;
- }
-
- int ret,i, rid = -1, nread = 0;
- while (1)
- {
- if ( args->tbx )
- {
- if ( (ret=tbx_itr_next(args->fp, args->tbx, args->itr, &args->tmps)) < 0 ) break; // no more lines
- ret = vcf_parse1(&args->tmps, args->hdr, args->rec);
- if ( ret<0 ) error("Could not parse the line: %s\n", args->tmps.s);
- }
- else if ( args->idx )
- {
- ret = bcf_itr_next(args->fp, args->itr, args->rec);
- if ( ret < -1 ) error("Could not parse a line from %s\n", reg);
- if ( ret < 0 ) break; // no more lines or an error
- }
- else
- {
- if ( args->fp->format.format==vcf )
- {
- if ( (ret=hts_getline(args->fp, KS_SEP_LINE, &args->tmps)) < 0 ) break; // no more lines
- ret = vcf_parse1(&args->tmps, args->hdr, args->rec);
- if ( ret<0 ) error("Could not parse the line: %s\n", args->tmps.s);
- }
- else if ( args->fp->format.format==bcf )
- {
- ret = bcf_read1(args->fp, args->hdr, args->rec);
- if ( ret < -1 ) error("Could not parse %s\n", args->fname);
- if ( ret < 0 ) break; // no more lines or an error
- }
- if ( rid!=-1 && rid!=args->rec->rid )
- {
- report(args, bcf_hdr_id2name(args->hdr,rid));
- nread = 0;
- }
- rid = args->rec->rid;
- }
-
- bcf_unpack(args->rec, BCF_UN_FMT);
- bcf_fmt_t *fmt_gt = NULL;
- for (i=0; i<args->rec->n_fmt; i++)
- if ( args->rec->d.fmt[i].id==args->gt_id ) { fmt_gt = &args->rec->d.fmt[i]; break; }
- if ( !fmt_gt ) continue; // no GT tag
- if ( fmt_gt->n==0 ) continue; // empty?!
- if ( fmt_gt->type!=BCF_BT_INT8 ) error("TODO: the GT fmt_type is not int8!\n");
-
- // update the array of missing samples
- for (i=0; i<args->nsmpl; i++)
- {
- int8_t *ptr = (int8_t*) (fmt_gt->p + args->smpl[i]*fmt_gt->size);
- int ial = 0;
- for (ial=0; ial<fmt_gt->n; ial++)
- if ( ptr[ial]==bcf_gt_missing || ptr[ial]==bcf_int8_vector_end ) break;
- if ( ial==0 ) continue; // missing
- if ( ++args->nsites[i] < args->min_sites ) continue;
- if ( i+1<args->nsmpl )
- {
- memmove(args->smpl+i, args->smpl+i+1, sizeof(int)*(args->nsmpl-i-1));
- memmove(args->nsites+i, args->nsites+i+1, sizeof(int)*(args->nsmpl-i-1));
- }
- args->nsmpl--;
- i--;
- }
- nread = 1;
- if ( !args->nsmpl ) break;
- }
- if ( nread ) report(args, rid==-1 ? reg : bcf_hdr_id2name(args->hdr,rid));
-
- tbx_itr_destroy(args->itr);
- args->itr = NULL;
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->min_sites = 1;
- static struct option loptions[] =
- {
- {"n-markers",required_argument,NULL,'n'},
- {"regions",required_argument,NULL,'r'},
- {"regions-file",required_argument,NULL,'R'},
- {NULL,0,NULL,0}
- };
- int c,i;
- char *tmp;
- while ((c = getopt_long(argc, argv, "vr:R:n:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'n':
- args->min_sites = strtol(optarg,&tmp,10);
- if ( *tmp ) error("Could not parse: -n %s\n", optarg);
- break;
- case 'R': args->region_is_file = 1;
- case 'r': args->region = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
-
- if ( optind>=argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else error("%s",usage_text());
- }
- else args->fname = argv[optind];
- init_data(args);
-
- for (i=0; i<args->nregs; i++) test_region(args, args->regs[i]);
- if ( !args->nregs ) test_region(args, NULL);
-
- destroy_data(args);
- free(args);
- return 0;
-}
-
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2015 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-/*
- Trio haplotypes: mother (A,B), father (C,D), child (E,F)
- Modeling the following states:
- 01|23|02
- 01|23|03
- 01|23|12
- 01|23|13
- 01|23|20
- 01|23|30
- 01|23|21
- 01|23|31
- with the likelihoods of two haplotypes A,B segments sharing an allele:
- P(01|A==B) .. e (P of error)
- P(00|A==B) .. 1-e
- and
- P(ab,cd,ef|E=A,F=C) = P(ea|E=A)*P(fc|F=C)
-
-
- Unrelated samples: (A,B) and (C,D)
- Modeling the states:
- xxxx .. A!=C,A!=D,B!=C,B!=D
- 0x0x .. A=C,B!=D
- 0xx0 .. A=D,B!=C
- x00x .. B=C,A!=D
- x0x0 .. B=D,A!=C
- 0101 .. A=C,B=D
- 0110 .. A=D,B=C
- with the likelihoods
- P(01|A!=B) .. f*(1-f)
- P(00|A!=B) .. (1-f)*(1-f)
- P(11|A!=B) .. f*f
-
- Assuming 2x30 crossovers, P=2e-8.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <errno.h>
-#include "bcftools.h"
-#include "HMM.h"
-
-#define C_TRIO 1
-#define C_UNRL 2
-
-// states for unrelated samples
-#define UNRL_xxxx 0
-#define UNRL_0x0x 1
-#define UNRL_0xx0 2
-#define UNRL_x00x 3
-#define UNRL_x0x0 4
-#define UNRL_0101 5
-#define UNRL_0110 6
-
-// trio states
-#define TRIO_AC 0
-#define TRIO_AD 1
-#define TRIO_BC 2
-#define TRIO_BD 3
-#define TRIO_CA 4
-#define TRIO_DA 5
-#define TRIO_CB 6
-#define TRIO_DB 7
-
-typedef struct _args_t
-{
- bcf_hdr_t *hdr;
- hmm_t *hmm;
- double *eprob, *tprob, pij, pgt_err;
- uint32_t *sites;
- int32_t *gt_arr;
- int nsites, msites, ngt_arr, prev_rid;
- int mode, nstates, nhet_father, nhet_mother;
- int imother,ifather,ichild, isample,jsample;
- void (*set_observed_prob) (bcf1_t *rec);
- char *prefix;
- FILE *fp;
-}
-args_t;
-
-static args_t args;
-
-#define SW_MOTHER 1
-#define SW_FATHER 2
-static int hap_switch[8][8];
-
-static void set_observed_prob_trio(bcf1_t *rec);
-static void set_observed_prob_unrelated(bcf1_t *rec);
-static void init_hmm_trio(args_t *args);
-static void init_hmm_unrelated(args_t *args);
-
-
-const char *about(void)
-{
- return "Color shared chromosomal segments, requires phased GTs.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Color shared chromosomal segments, requires phased GTs. The output\n"
- " can be visualized using the color-chrs.pl script.\n"
- "Usage: bcftools +color-chrs [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -p, --prefix <path> output files prefix\n"
- " -t, --trio <m,f,c> names of mother, father and the child\n"
- " -u, --unrelated <a,b> names of two unrelated samples\n"
- "\n"
- "Example:\n"
- " bcftools +color-chrs in.vcf --\n"
- "\n";
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- char *trio_samples = NULL, *unrelated_samples = NULL;
- memset(&args,0,sizeof(args_t));
- args.prev_rid = -1;
- args.hdr = in;
- args.pij = 2e-8;
- args.pgt_err = 1e-9;
-
- static struct option loptions[] =
- {
- {"prefix",1,0,'p'},
- {"trio",1,0,'t'},
- {"unrelated",1,0,'u'},
- {0,0,0,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?ht:u:p:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'p': args.prefix = optarg; break;
- case 't': trio_samples = optarg; break;
- case 'u': unrelated_samples = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( optind != argc ) error("%s",usage());
- if ( trio_samples && unrelated_samples ) error("Expected only one of the -t/-u options\n");
- if ( !trio_samples && !unrelated_samples ) error("Expected one of the -t/-u options\n");
- if ( !args.prefix ) error("Expected the -p option\n");
-
- int ret = bcf_hdr_set_samples(args.hdr, trio_samples ? trio_samples : unrelated_samples, 0);
- if ( ret<0 ) error("Could not parse samples: %s\n", trio_samples ? trio_samples : unrelated_samples);
- else if ( ret>0 ) error("%d-th sample not found: %s\n", ret,trio_samples ? trio_samples : unrelated_samples);
-
- if ( trio_samples )
- {
- int i,n = 0;
- char **list = hts_readlist(trio_samples, 0, &n);
- if ( n!=3 ) error("Expected three sample names with -t\n");
- args.imother = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[0]);
- args.ifather = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[1]);
- args.ichild = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[2]);
- for (i=0; i<n; i++) free(list[i]);
- free(list);
- args.set_observed_prob = set_observed_prob_trio;
- args.mode = C_TRIO;
- init_hmm_trio(&args);
- }
- else
- {
- int i,n = 0;
- char **list = hts_readlist(unrelated_samples, 0, &n);
- if ( n!=2 ) error("Expected two sample names with -u\n");
- args.isample = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[0]);
- args.jsample = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[1]);
- for (i=0; i<n; i++) free(list[i]);
- free(list);
- args.set_observed_prob = set_observed_prob_unrelated;
- args.mode = C_UNRL;
- init_hmm_unrelated(&args);
- }
- return 1;
-}
-
-static void init_hmm_trio(args_t *args)
-{
- int i,j;
- args->nstates = 8;
- args->tprob = (double*) malloc(sizeof(double)*args->nstates*args->nstates);
-
- for (i=0; i<args->nstates; i++)
- for (j=0; j<args->nstates; j++) hap_switch[i][j] = 0;
-
- hap_switch[TRIO_AD][TRIO_AC] = SW_FATHER;
- hap_switch[TRIO_BC][TRIO_AC] = SW_MOTHER;
- hap_switch[TRIO_BD][TRIO_AC] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_AC][TRIO_AD] = SW_FATHER;
- hap_switch[TRIO_BC][TRIO_AD] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_BD][TRIO_AD] = SW_MOTHER;
- hap_switch[TRIO_AC][TRIO_BC] = SW_MOTHER;
- hap_switch[TRIO_AD][TRIO_BC] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_BD][TRIO_BC] = SW_FATHER;
- hap_switch[TRIO_AC][TRIO_BD] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_AD][TRIO_BD] = SW_MOTHER;
- hap_switch[TRIO_BC][TRIO_BD] = SW_FATHER;
-
- hap_switch[TRIO_DA][TRIO_CA] = SW_FATHER;
- hap_switch[TRIO_CB][TRIO_CA] = SW_MOTHER;
- hap_switch[TRIO_DB][TRIO_CA] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_CA][TRIO_DA] = SW_FATHER;
- hap_switch[TRIO_CB][TRIO_DA] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_DB][TRIO_DA] = SW_MOTHER;
- hap_switch[TRIO_CA][TRIO_CB] = SW_MOTHER;
- hap_switch[TRIO_DA][TRIO_CB] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_DB][TRIO_CB] = SW_FATHER;
- hap_switch[TRIO_CA][TRIO_DB] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_DA][TRIO_DB] = SW_MOTHER;
- hap_switch[TRIO_CB][TRIO_DB] = SW_FATHER;
-
- for (i=0; i<args->nstates; i++)
- {
- for (j=0; j<args->nstates; j++)
- {
- if ( !hap_switch[i][j] ) MAT(args->tprob,args->nstates,i,j) = 0;
- else
- {
- MAT(args->tprob,args->nstates,i,j) = 1;
- if ( hap_switch[i][j] & SW_MOTHER ) MAT(args->tprob,args->nstates,i,j) *= args->pij;
- if ( hap_switch[i][j] & SW_FATHER ) MAT(args->tprob,args->nstates,i,j) *= args->pij;
- }
- }
- }
- for (i=0; i<args->nstates; i++)
- {
- double sum = 0;
- for (j=0; j<args->nstates; j++)
- {
- if ( i!=j ) sum += MAT(args->tprob,args->nstates,i,j);
- }
- MAT(args->tprob,args->nstates,i,i) = 1 - sum;
- }
-
- #if 0
- for (i=0; i<args->nstates; i++)
- {
- for (j=0; j<args->nstates; j++)
- fprintf(stderr,"\t%d",hap_switch[j][i]);
- fprintf(stderr,"\n");
- }
- for (i=0; i<args->nstates; i++)
- {
- for (j=0; j<args->nstates; j++)
- fprintf(stderr,"\t%e",MAT(args->tprob,args->nstates,j,i));
- fprintf(stderr,"\n");
- }
- #endif
-
- args->hmm = hmm_init(args->nstates, args->tprob, 10000);
-}
-static void init_hmm_unrelated(args_t *args)
-{
- int i,j;
- args->nstates = 7;
- args->tprob = (double*) malloc(sizeof(double)*args->nstates*args->nstates);
-
- for (i=0; i<args->nstates; i++)
- {
- for (j=0; j<args->nstates; j++)
- MAT(args->tprob,args->nstates,i,j) = args->pij;
- }
- MAT(args->tprob,args->nstates,UNRL_0101,UNRL_xxxx) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0110,UNRL_xxxx) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_x0x0,UNRL_0x0x) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0110,UNRL_0x0x) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_x00x,UNRL_0xx0) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0101,UNRL_0xx0) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0101,UNRL_x00x) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0110,UNRL_x0x0) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0110,UNRL_0101) = args->pij*args->pij;
-
- for (i=0; i<args->nstates; i++)
- {
- for (j=i+1; j<args->nstates; j++)
- MAT(args->tprob,args->nstates,i,j) = MAT(args->tprob,args->nstates,j,i);
- }
- for (i=0; i<args->nstates; i++)
- {
- double sum = 0;
- for (j=0; j<args->nstates; j++)
- if ( i!=j ) sum += MAT(args->tprob,args->nstates,i,j);
- MAT(args->tprob,args->nstates,i,i) = 1 - sum;
- }
-
- #if 0
- for (i=0; i<args->nstates; i++)
- {
- for (j=0; j<args->nstates; j++)
- fprintf(stderr,"\t%e",MAT(args->tprob,args->nstates,j,i));
- fprintf(stderr,"\n");
- }
- #endif
-
- args->hmm = hmm_init(args->nstates, args->tprob, 10000);
-}
-static inline double prob_shared(float af, int a, int b)
-{
- return a==b ? 1-args.pgt_err : args.pgt_err;
-}
-static inline double prob_not_shared(float af, int a, int b)
-{
- if ( a!=b ) return af*(1-af);
- else if ( a==0 ) return (1-af)*(1-af);
- else return af*af;
-}
-static void set_observed_prob_unrelated(bcf1_t *rec)
-{
- float af = 0.5; // alternate allele frequency
-
- int ngt = bcf_get_genotypes(args.hdr, rec, &args.gt_arr, &args.ngt_arr);
- if ( ngt<0 ) return;
- if ( ngt!=4 ) return; // chrX
-
- int32_t a,b,c,d;
- a = args.gt_arr[2*args.isample];
- b = args.gt_arr[2*args.isample+1];
- c = args.gt_arr[2*args.jsample];
- d = args.gt_arr[2*args.jsample+1];
- if ( bcf_gt_is_missing(a) || bcf_gt_is_missing(b) ) return;
- if ( bcf_gt_is_missing(c) || bcf_gt_is_missing(d) ) return;
- if ( !bcf_gt_is_phased(a) && !bcf_gt_is_phased(b) ) return; // only the second allele should be set when phased
- if ( !bcf_gt_is_phased(c) && !bcf_gt_is_phased(d) ) return;
- a = bcf_gt_allele(a);
- b = bcf_gt_allele(b);
- c = bcf_gt_allele(c);
- d = bcf_gt_allele(d);
-
- int m = args.msites;
- args.nsites++;
- hts_expand(uint32_t,args.nsites,args.msites,args.sites);
- if ( m!=args.msites ) args.eprob = (double*) realloc(args.eprob, sizeof(double)*args.msites*args.nstates);
-
- args.sites[args.nsites-1] = rec->pos;
- double *prob = args.eprob + args.nstates*(args.nsites-1);
- prob[UNRL_xxxx] = prob_not_shared(af,a,c) * prob_not_shared(af,a,d) * prob_not_shared(af,b,c) * prob_not_shared(af,b,d);
- prob[UNRL_0x0x] = prob_shared(af,a,c) * prob_not_shared(af,b,d);
- prob[UNRL_0xx0] = prob_shared(af,a,d) * prob_not_shared(af,b,c);
- prob[UNRL_x00x] = prob_shared(af,b,c) * prob_not_shared(af,a,d);
- prob[UNRL_x0x0] = prob_shared(af,b,d) * prob_not_shared(af,a,c);
- prob[UNRL_0101] = prob_shared(af,a,c) * prob_shared(af,b,d);
- prob[UNRL_0110] = prob_shared(af,a,d) * prob_shared(af,b,c);
-
-#if 0
- static int x = 0;
- if ( !x++)
- {
- printf("p(0==0) .. %f\n", prob_shared(af,0,0));
- printf("p(0!=0) .. %f\n", prob_not_shared(af,0,0));
- printf("p(0==1) .. %f\n", prob_shared(af,0,1));
- printf("p(0!=1) .. %f\n", prob_not_shared(af,0,1));
- }
- printf("%d|%d %d|%d x:%f 11:%f 12:%f 21:%f 22:%f 11,22:%f 12,21:%f %d\n", a,b,c,d,
- prob[UNRL_xxxx], prob[UNRL_0x0x], prob[UNRL_0xx0], prob[UNRL_x00x], prob[UNRL_x0x0], prob[UNRL_0101], prob[UNRL_0110], rec->pos+1);
-#endif
-}
-static void set_observed_prob_trio(bcf1_t *rec)
-{
- int ngt = bcf_get_genotypes(args.hdr, rec, &args.gt_arr, &args.ngt_arr);
- if ( ngt<0 ) return;
- if ( ngt!=6 ) return; // chrX
-
- int32_t a,b,c,d,e,f;
- a = args.gt_arr[2*args.imother];
- b = args.gt_arr[2*args.imother+1];
- c = args.gt_arr[2*args.ifather];
- d = args.gt_arr[2*args.ifather+1];
- e = args.gt_arr[2*args.ichild];
- f = args.gt_arr[2*args.ichild+1];
- if ( bcf_gt_is_missing(a) || bcf_gt_is_missing(b) ) return;
- if ( bcf_gt_is_missing(c) || bcf_gt_is_missing(d) ) return;
- if ( bcf_gt_is_missing(e) || bcf_gt_is_missing(f) ) return;
- if ( !bcf_gt_is_phased(a) && !bcf_gt_is_phased(b) ) return; // only the second allele should be set when phased
- if ( !bcf_gt_is_phased(c) && !bcf_gt_is_phased(d) ) return;
- if ( !bcf_gt_is_phased(e) && !bcf_gt_is_phased(f) ) return;
- a = bcf_gt_allele(a);
- b = bcf_gt_allele(b);
- c = bcf_gt_allele(c);
- d = bcf_gt_allele(d);
- e = bcf_gt_allele(e);
- f = bcf_gt_allele(f);
-
- int mother = (1<<a) | (1<<b);
- int father = (1<<c) | (1<<d);
- int child = (1<<e) | (1<<f);
- if ( !(mother&child) || !(father&child) ) return; // Mendelian-inconsistent site, skip
-
- if ( a!=b ) args.nhet_mother++;
- if ( c!=d ) args.nhet_father++;
-
- int m = args.msites;
- args.nsites++;
- hts_expand(uint32_t,args.nsites,args.msites,args.sites);
- if ( m!=args.msites ) args.eprob = (double*) realloc(args.eprob, sizeof(double)*args.msites*args.nstates);
-
- args.sites[args.nsites-1] = rec->pos;
- double *prob = args.eprob + args.nstates*(args.nsites-1);
- prob[TRIO_AC] = prob_shared(0,e,a) * prob_shared(0,f,c);
- prob[TRIO_AD] = prob_shared(0,e,a) * prob_shared(0,f,d);
- prob[TRIO_BC] = prob_shared(0,e,b) * prob_shared(0,f,c);
- prob[TRIO_BD] = prob_shared(0,e,b) * prob_shared(0,f,d);
- prob[TRIO_CA] = prob_shared(0,e,c) * prob_shared(0,f,a);
- prob[TRIO_DA] = prob_shared(0,e,d) * prob_shared(0,f,a);
- prob[TRIO_CB] = prob_shared(0,e,c) * prob_shared(0,f,b);
- prob[TRIO_DB] = prob_shared(0,e,d) * prob_shared(0,f,b);
-}
-
-void flush_viterbi(args_t *args)
-{
- const char *s1, *s2, *s3 = NULL;
- if ( args->mode==C_UNRL )
- {
- s1 = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,args->isample);
- s2 = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,args->jsample);
- }
- else if ( args->mode==C_TRIO )
- {
- s1 = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,args->imother);
- s3 = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,args->ifather);
- s2 = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,args->ichild);
- }
- else abort();
-
- if ( !args->fp )
- {
- kstring_t str = {0,0,0};
- kputs(args->prefix, &str);
- kputs(".dat", &str);
- args->fp = fopen(str.s,"w");
- if ( !args->fp ) error("%s: %s\n", str.s,strerror(errno));
- free(str.s);
- fprintf(args->fp,"# SG, shared segment\t[2]Chromosome\t[3]Start\t[4]End\t[5]%s:1\t[6]%s:2\n",s2,s2);
- fprintf(args->fp,"# SW, number of switches\t[3]Sample\t[4]Chromosome\t[5]nHets\t[5]nSwitches\t[6]switch rate\n");
- }
-
- hmm_run_viterbi(args->hmm,args->nsites,args->eprob,args->sites);
- uint8_t *vpath = hmm_get_viterbi_path(args->hmm);
- int i, iprev = -1, prev_state = -1, nstates = hmm_get_nstates(args->hmm);
- int nswitch_mother = 0, nswitch_father = 0;
- for (i=0; i<args->nsites; i++)
- {
- int state = vpath[i*nstates];
- if ( state!=prev_state || i+1==args->nsites )
- {
- uint32_t start = iprev>=0 ? args->sites[iprev]+1 : 1, end = i>0 ? args->sites[i-1] : 1;
- const char *chr = bcf_hdr_id2name(args->hdr,args->prev_rid);
- if ( args->mode==C_UNRL )
- {
- switch (prev_state)
- {
- case UNRL_0x0x:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t-\n", chr,start,end,s1); break;
- case UNRL_0xx0:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t-\t%s:1\n", chr,start,end,s1); break;
- case UNRL_x00x:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t-\n", chr,start,end,s1); break;
- case UNRL_x0x0:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t-\t%s:2\n", chr,start,end,s1); break;
- case UNRL_0101:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t%s:2\n", chr,start,end,s1,s1); break;
- case UNRL_0110:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t%s:1\n", chr,start,end,s1,s1); break;
- }
- }
- else if ( args->mode==C_TRIO )
- {
- switch (prev_state)
- {
- case TRIO_AC:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t%s:1\n", chr,start,end,s1,s3); break;
- case TRIO_AD:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t%s:2\n", chr,start,end,s1,s3); break;
- case TRIO_BC:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t%s:1\n", chr,start,end,s1,s3); break;
- case TRIO_BD:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t%s:2\n", chr,start,end,s1,s3); break;
- case TRIO_CA:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t%s:1\n", chr,start,end,s3,s1); break;
- case TRIO_DA:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t%s:1\n", chr,start,end,s3,s1); break;
- case TRIO_CB:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t%s:2\n", chr,start,end,s3,s1); break;
- case TRIO_DB:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t%s:2\n", chr,start,end,s3,s1); break;
- }
- if ( hap_switch[state][prev_state] & SW_MOTHER ) nswitch_mother++;
- if ( hap_switch[state][prev_state] & SW_FATHER ) nswitch_father++;
- }
- iprev = i-1;
- }
- prev_state = state;
- }
- float mrate = args->nhet_mother>1 ? (float)nswitch_mother/(args->nhet_mother-1) : 0;
- float frate = args->nhet_father>1 ? (float)nswitch_father/(args->nhet_father-1) : 0;
- fprintf(args->fp,"SW\t%s\t%s\t%d\t%d\t%f\n", s1,bcf_hdr_id2name(args->hdr,args->prev_rid),args->nhet_mother,nswitch_mother,mrate);
- fprintf(args->fp,"SW\t%s\t%s\t%d\t%d\t%f\n", s3,bcf_hdr_id2name(args->hdr,args->prev_rid),args->nhet_father,nswitch_father,frate);
- args->nsites = 0;
- args->nhet_father = args->nhet_mother = 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- if ( args.prev_rid==-1 ) args.prev_rid = rec->rid;
- if ( args.prev_rid!=rec->rid ) flush_viterbi(&args);
- args.prev_rid = rec->rid;
- args.set_observed_prob(rec);
- return NULL;
-}
-
-void destroy(void)
-{
- flush_viterbi(&args);
- fclose(args.fp);
-
- free(args.gt_arr);
- free(args.tprob);
- free(args.sites);
- free(args.eprob);
- hmm_destroy(args.hmm);
-}
-
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2015 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-/*
- Trio haplotypes: mother (A,B), father (C,D), child (E,F)
- Modeling the following states:
- 01|23|02
- 01|23|03
- 01|23|12
- 01|23|13
- 01|23|20
- 01|23|30
- 01|23|21
- 01|23|31
- with the likelihoods of two haplotypes A,B segments sharing an allele:
- P(01|A==B) .. e (P of error)
- P(00|A==B) .. 1-e
- and
- P(ab,cd,ef|E=A,F=C) = P(ea|E=A)*P(fc|F=C)
-
-
- Unrelated samples: (A,B) and (C,D)
- Modeling the states:
- xxxx .. A!=C,A!=D,B!=C,B!=D
- 0x0x .. A=C,B!=D
- 0xx0 .. A=D,B!=C
- x00x .. B=C,A!=D
- x0x0 .. B=D,A!=C
- 0101 .. A=C,B=D
- 0110 .. A=D,B=C
- with the likelihoods
- P(01|A!=B) .. f*(1-f)
- P(00|A!=B) .. (1-f)*(1-f)
- P(11|A!=B) .. f*f
-
- Assuming 2x30 crossovers, P=2e-8.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <errno.h>
-#include "bcftools.h"
-#include "HMM.h"
-
-#define C_TRIO 1
-#define C_UNRL 2
-
-// states for unrelated samples
-#define UNRL_xxxx 0
-#define UNRL_0x0x 1
-#define UNRL_0xx0 2
-#define UNRL_x00x 3
-#define UNRL_x0x0 4
-#define UNRL_0101 5
-#define UNRL_0110 6
-
-// trio states
-#define TRIO_AC 0
-#define TRIO_AD 1
-#define TRIO_BC 2
-#define TRIO_BD 3
-#define TRIO_CA 4
-#define TRIO_DA 5
-#define TRIO_CB 6
-#define TRIO_DB 7
-
-typedef struct _args_t
-{
- bcf_hdr_t *hdr;
- hmm_t *hmm;
- double *eprob, *tprob, pij, pgt_err;
- uint32_t *sites;
- int32_t *gt_arr;
- int nsites, msites, ngt_arr, prev_rid;
- int mode, nstates, nhet_father, nhet_mother;
- int imother,ifather,ichild, isample,jsample;
- void (*set_observed_prob) (bcf1_t *rec);
- char *prefix;
- FILE *fp;
-}
-args_t;
-
-static args_t args;
-
-#define SW_MOTHER 1
-#define SW_FATHER 2
-static int hap_switch[8][8];
-
-static void set_observed_prob_trio(bcf1_t *rec);
-static void set_observed_prob_unrelated(bcf1_t *rec);
-static void init_hmm_trio(args_t *args);
-static void init_hmm_unrelated(args_t *args);
-
-
-const char *about(void)
-{
- return "Color shared chromosomal segments, requires phased GTs.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Color shared chromosomal segments, requires phased GTs. The output\n"
- " can be visualized using the color-chrs.pl script.\n"
- "Usage: bcftools +color-chrs [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -p, --prefix <path> output files prefix\n"
- " -t, --trio <m,f,c> names of mother, father and the child\n"
- " -u, --unrelated <a,b> names of two unrelated samples\n"
- "\n"
- "Example:\n"
- " bcftools +color-chrs in.vcf --\n"
- "\n";
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- char *trio_samples = NULL, *unrelated_samples = NULL;
- memset(&args,0,sizeof(args_t));
- args.prev_rid = -1;
- args.hdr = in;
- args.pij = 2e-8;
- args.pgt_err = 1e-9;
-
- static struct option loptions[] =
- {
- {"prefix",1,0,'p'},
- {"trio",1,0,'t'},
- {"unrelated",1,0,'u'},
- {0,0,0,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?ht:u:p:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'p': args.prefix = optarg; break;
- case 't': trio_samples = optarg; break;
- case 'u': unrelated_samples = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( optind != argc ) error("%s",usage());
- if ( trio_samples && unrelated_samples ) error("Expected only one of the -t/-u options\n");
- if ( !trio_samples && !unrelated_samples ) error("Expected one of the -t/-u options\n");
- if ( !args.prefix ) error("Expected the -p option\n");
-
- int ret = bcf_hdr_set_samples(args.hdr, trio_samples ? trio_samples : unrelated_samples, 0);
- if ( ret<0 ) error("Could not parse samples: %s\n", trio_samples ? trio_samples : unrelated_samples);
- else if ( ret>0 ) error("%d-th sample not found: %s\n", ret,trio_samples ? trio_samples : unrelated_samples);
-
- if ( trio_samples )
- {
- int i,n = 0;
- char **list = hts_readlist(trio_samples, 0, &n);
- if ( n!=3 ) error("Expected three sample names with -t\n");
- args.imother = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[0]);
- args.ifather = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[1]);
- args.ichild = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[2]);
- for (i=0; i<n; i++) free(list[i]);
- free(list);
- args.set_observed_prob = set_observed_prob_trio;
- args.mode = C_TRIO;
- init_hmm_trio(&args);
- }
- else
- {
- int i,n = 0;
- char **list = hts_readlist(unrelated_samples, 0, &n);
- if ( n!=2 ) error("Expected two sample names with -u\n");
- args.isample = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[0]);
- args.jsample = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[1]);
- for (i=0; i<n; i++) free(list[i]);
- free(list);
- args.set_observed_prob = set_observed_prob_unrelated;
- args.mode = C_UNRL;
- init_hmm_unrelated(&args);
- }
- return 1;
-}
-
-static void init_hmm_trio(args_t *args)
-{
- int i,j;
- args->nstates = 8;
- args->tprob = (double*) malloc(sizeof(double)*args->nstates*args->nstates);
-
- for (i=0; i<args->nstates; i++)
- for (j=0; j<args->nstates; j++) hap_switch[i][j] = 0;
-
- hap_switch[TRIO_AD][TRIO_AC] = SW_FATHER;
- hap_switch[TRIO_BC][TRIO_AC] = SW_MOTHER;
- hap_switch[TRIO_BD][TRIO_AC] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_AC][TRIO_AD] = SW_FATHER;
- hap_switch[TRIO_BC][TRIO_AD] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_BD][TRIO_AD] = SW_MOTHER;
- hap_switch[TRIO_AC][TRIO_BC] = SW_MOTHER;
- hap_switch[TRIO_AD][TRIO_BC] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_BD][TRIO_BC] = SW_FATHER;
- hap_switch[TRIO_AC][TRIO_BD] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_AD][TRIO_BD] = SW_MOTHER;
- hap_switch[TRIO_BC][TRIO_BD] = SW_FATHER;
-
- hap_switch[TRIO_DA][TRIO_CA] = SW_FATHER;
- hap_switch[TRIO_CB][TRIO_CA] = SW_MOTHER;
- hap_switch[TRIO_DB][TRIO_CA] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_CA][TRIO_DA] = SW_FATHER;
- hap_switch[TRIO_CB][TRIO_DA] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_DB][TRIO_DA] = SW_MOTHER;
- hap_switch[TRIO_CA][TRIO_CB] = SW_MOTHER;
- hap_switch[TRIO_DA][TRIO_CB] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_DB][TRIO_CB] = SW_FATHER;
- hap_switch[TRIO_CA][TRIO_DB] = SW_MOTHER | SW_FATHER;
- hap_switch[TRIO_DA][TRIO_DB] = SW_MOTHER;
- hap_switch[TRIO_CB][TRIO_DB] = SW_FATHER;
-
- for (i=0; i<args->nstates; i++)
- {
- for (j=0; j<args->nstates; j++)
- {
- if ( !hap_switch[i][j] ) MAT(args->tprob,args->nstates,i,j) = 0;
- else
- {
- MAT(args->tprob,args->nstates,i,j) = 1;
- if ( hap_switch[i][j] & SW_MOTHER ) MAT(args->tprob,args->nstates,i,j) *= args->pij;
- if ( hap_switch[i][j] & SW_FATHER ) MAT(args->tprob,args->nstates,i,j) *= args->pij;
- }
- }
- }
- for (i=0; i<args->nstates; i++)
- {
- double sum = 0;
- for (j=0; j<args->nstates; j++)
- {
- if ( i!=j ) sum += MAT(args->tprob,args->nstates,i,j);
- }
- MAT(args->tprob,args->nstates,i,i) = 1 - sum;
- }
-
- #if 0
- for (i=0; i<args->nstates; i++)
- {
- for (j=0; j<args->nstates; j++)
- fprintf(bcftools_stderr,"\t%d",hap_switch[j][i]);
- fprintf(bcftools_stderr,"\n");
- }
- for (i=0; i<args->nstates; i++)
- {
- for (j=0; j<args->nstates; j++)
- fprintf(bcftools_stderr,"\t%e",MAT(args->tprob,args->nstates,j,i));
- fprintf(bcftools_stderr,"\n");
- }
- #endif
-
- args->hmm = hmm_init(args->nstates, args->tprob, 10000);
-}
-static void init_hmm_unrelated(args_t *args)
-{
- int i,j;
- args->nstates = 7;
- args->tprob = (double*) malloc(sizeof(double)*args->nstates*args->nstates);
-
- for (i=0; i<args->nstates; i++)
- {
- for (j=0; j<args->nstates; j++)
- MAT(args->tprob,args->nstates,i,j) = args->pij;
- }
- MAT(args->tprob,args->nstates,UNRL_0101,UNRL_xxxx) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0110,UNRL_xxxx) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_x0x0,UNRL_0x0x) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0110,UNRL_0x0x) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_x00x,UNRL_0xx0) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0101,UNRL_0xx0) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0101,UNRL_x00x) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0110,UNRL_x0x0) = args->pij*args->pij;
- MAT(args->tprob,args->nstates,UNRL_0110,UNRL_0101) = args->pij*args->pij;
-
- for (i=0; i<args->nstates; i++)
- {
- for (j=i+1; j<args->nstates; j++)
- MAT(args->tprob,args->nstates,i,j) = MAT(args->tprob,args->nstates,j,i);
- }
- for (i=0; i<args->nstates; i++)
- {
- double sum = 0;
- for (j=0; j<args->nstates; j++)
- if ( i!=j ) sum += MAT(args->tprob,args->nstates,i,j);
- MAT(args->tprob,args->nstates,i,i) = 1 - sum;
- }
-
- #if 0
- for (i=0; i<args->nstates; i++)
- {
- for (j=0; j<args->nstates; j++)
- fprintf(bcftools_stderr,"\t%e",MAT(args->tprob,args->nstates,j,i));
- fprintf(bcftools_stderr,"\n");
- }
- #endif
-
- args->hmm = hmm_init(args->nstates, args->tprob, 10000);
-}
-static inline double prob_shared(float af, int a, int b)
-{
- return a==b ? 1-args.pgt_err : args.pgt_err;
-}
-static inline double prob_not_shared(float af, int a, int b)
-{
- if ( a!=b ) return af*(1-af);
- else if ( a==0 ) return (1-af)*(1-af);
- else return af*af;
-}
-static void set_observed_prob_unrelated(bcf1_t *rec)
-{
- float af = 0.5; // alternate allele frequency
-
- int ngt = bcf_get_genotypes(args.hdr, rec, &args.gt_arr, &args.ngt_arr);
- if ( ngt<0 ) return;
- if ( ngt!=4 ) return; // chrX
-
- int32_t a,b,c,d;
- a = args.gt_arr[2*args.isample];
- b = args.gt_arr[2*args.isample+1];
- c = args.gt_arr[2*args.jsample];
- d = args.gt_arr[2*args.jsample+1];
- if ( bcf_gt_is_missing(a) || bcf_gt_is_missing(b) ) return;
- if ( bcf_gt_is_missing(c) || bcf_gt_is_missing(d) ) return;
- if ( !bcf_gt_is_phased(a) && !bcf_gt_is_phased(b) ) return; // only the second allele should be set when phased
- if ( !bcf_gt_is_phased(c) && !bcf_gt_is_phased(d) ) return;
- a = bcf_gt_allele(a);
- b = bcf_gt_allele(b);
- c = bcf_gt_allele(c);
- d = bcf_gt_allele(d);
-
- int m = args.msites;
- args.nsites++;
- hts_expand(uint32_t,args.nsites,args.msites,args.sites);
- if ( m!=args.msites ) args.eprob = (double*) realloc(args.eprob, sizeof(double)*args.msites*args.nstates);
-
- args.sites[args.nsites-1] = rec->pos;
- double *prob = args.eprob + args.nstates*(args.nsites-1);
- prob[UNRL_xxxx] = prob_not_shared(af,a,c) * prob_not_shared(af,a,d) * prob_not_shared(af,b,c) * prob_not_shared(af,b,d);
- prob[UNRL_0x0x] = prob_shared(af,a,c) * prob_not_shared(af,b,d);
- prob[UNRL_0xx0] = prob_shared(af,a,d) * prob_not_shared(af,b,c);
- prob[UNRL_x00x] = prob_shared(af,b,c) * prob_not_shared(af,a,d);
- prob[UNRL_x0x0] = prob_shared(af,b,d) * prob_not_shared(af,a,c);
- prob[UNRL_0101] = prob_shared(af,a,c) * prob_shared(af,b,d);
- prob[UNRL_0110] = prob_shared(af,a,d) * prob_shared(af,b,c);
-
-#if 0
- static int x = 0;
- if ( !x++)
- {
- fprintf(bcftools_stdout, "p(0==0) .. %f\n", prob_shared(af,0,0));
- fprintf(bcftools_stdout, "p(0!=0) .. %f\n", prob_not_shared(af,0,0));
- fprintf(bcftools_stdout, "p(0==1) .. %f\n", prob_shared(af,0,1));
- fprintf(bcftools_stdout, "p(0!=1) .. %f\n", prob_not_shared(af,0,1));
- }
- fprintf(bcftools_stdout, "%d|%d %d|%d x:%f 11:%f 12:%f 21:%f 22:%f 11,22:%f 12,21:%f %d\n", a,b,c,d,
- prob[UNRL_xxxx], prob[UNRL_0x0x], prob[UNRL_0xx0], prob[UNRL_x00x], prob[UNRL_x0x0], prob[UNRL_0101], prob[UNRL_0110], rec->pos+1);
-#endif
-}
-static void set_observed_prob_trio(bcf1_t *rec)
-{
- int ngt = bcf_get_genotypes(args.hdr, rec, &args.gt_arr, &args.ngt_arr);
- if ( ngt<0 ) return;
- if ( ngt!=6 ) return; // chrX
-
- int32_t a,b,c,d,e,f;
- a = args.gt_arr[2*args.imother];
- b = args.gt_arr[2*args.imother+1];
- c = args.gt_arr[2*args.ifather];
- d = args.gt_arr[2*args.ifather+1];
- e = args.gt_arr[2*args.ichild];
- f = args.gt_arr[2*args.ichild+1];
- if ( bcf_gt_is_missing(a) || bcf_gt_is_missing(b) ) return;
- if ( bcf_gt_is_missing(c) || bcf_gt_is_missing(d) ) return;
- if ( bcf_gt_is_missing(e) || bcf_gt_is_missing(f) ) return;
- if ( !bcf_gt_is_phased(a) && !bcf_gt_is_phased(b) ) return; // only the second allele should be set when phased
- if ( !bcf_gt_is_phased(c) && !bcf_gt_is_phased(d) ) return;
- if ( !bcf_gt_is_phased(e) && !bcf_gt_is_phased(f) ) return;
- a = bcf_gt_allele(a);
- b = bcf_gt_allele(b);
- c = bcf_gt_allele(c);
- d = bcf_gt_allele(d);
- e = bcf_gt_allele(e);
- f = bcf_gt_allele(f);
-
- int mother = (1<<a) | (1<<b);
- int father = (1<<c) | (1<<d);
- int child = (1<<e) | (1<<f);
- if ( !(mother&child) || !(father&child) ) return; // Mendelian-inconsistent site, skip
-
- if ( a!=b ) args.nhet_mother++;
- if ( c!=d ) args.nhet_father++;
-
- int m = args.msites;
- args.nsites++;
- hts_expand(uint32_t,args.nsites,args.msites,args.sites);
- if ( m!=args.msites ) args.eprob = (double*) realloc(args.eprob, sizeof(double)*args.msites*args.nstates);
-
- args.sites[args.nsites-1] = rec->pos;
- double *prob = args.eprob + args.nstates*(args.nsites-1);
- prob[TRIO_AC] = prob_shared(0,e,a) * prob_shared(0,f,c);
- prob[TRIO_AD] = prob_shared(0,e,a) * prob_shared(0,f,d);
- prob[TRIO_BC] = prob_shared(0,e,b) * prob_shared(0,f,c);
- prob[TRIO_BD] = prob_shared(0,e,b) * prob_shared(0,f,d);
- prob[TRIO_CA] = prob_shared(0,e,c) * prob_shared(0,f,a);
- prob[TRIO_DA] = prob_shared(0,e,d) * prob_shared(0,f,a);
- prob[TRIO_CB] = prob_shared(0,e,c) * prob_shared(0,f,b);
- prob[TRIO_DB] = prob_shared(0,e,d) * prob_shared(0,f,b);
-}
-
-void flush_viterbi(args_t *args)
-{
- const char *s1, *s2, *s3 = NULL;
- if ( args->mode==C_UNRL )
- {
- s1 = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,args->isample);
- s2 = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,args->jsample);
- }
- else if ( args->mode==C_TRIO )
- {
- s1 = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,args->imother);
- s3 = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,args->ifather);
- s2 = bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,args->ichild);
- }
- else abort();
-
- if ( !args->fp )
- {
- kstring_t str = {0,0,0};
- kputs(args->prefix, &str);
- kputs(".dat", &str);
- args->fp = fopen(str.s,"w");
- if ( !args->fp ) error("%s: %s\n", str.s,strerror(errno));
- free(str.s);
- fprintf(args->fp,"# SG, shared segment\t[2]Chromosome\t[3]Start\t[4]End\t[5]%s:1\t[6]%s:2\n",s2,s2);
- fprintf(args->fp,"# SW, number of switches\t[3]Sample\t[4]Chromosome\t[5]nHets\t[5]nSwitches\t[6]switch rate\n");
- }
-
- hmm_run_viterbi(args->hmm,args->nsites,args->eprob,args->sites);
- uint8_t *vpath = hmm_get_viterbi_path(args->hmm);
- int i, iprev = -1, prev_state = -1, nstates = hmm_get_nstates(args->hmm);
- int nswitch_mother = 0, nswitch_father = 0;
- for (i=0; i<args->nsites; i++)
- {
- int state = vpath[i*nstates];
- if ( state!=prev_state || i+1==args->nsites )
- {
- uint32_t start = iprev>=0 ? args->sites[iprev]+1 : 1, end = i>0 ? args->sites[i-1] : 1;
- const char *chr = bcf_hdr_id2name(args->hdr,args->prev_rid);
- if ( args->mode==C_UNRL )
- {
- switch (prev_state)
- {
- case UNRL_0x0x:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t-\n", chr,start,end,s1); break;
- case UNRL_0xx0:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t-\t%s:1\n", chr,start,end,s1); break;
- case UNRL_x00x:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t-\n", chr,start,end,s1); break;
- case UNRL_x0x0:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t-\t%s:2\n", chr,start,end,s1); break;
- case UNRL_0101:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t%s:2\n", chr,start,end,s1,s1); break;
- case UNRL_0110:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t%s:1\n", chr,start,end,s1,s1); break;
- }
- }
- else if ( args->mode==C_TRIO )
- {
- switch (prev_state)
- {
- case TRIO_AC:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t%s:1\n", chr,start,end,s1,s3); break;
- case TRIO_AD:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t%s:2\n", chr,start,end,s1,s3); break;
- case TRIO_BC:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t%s:1\n", chr,start,end,s1,s3); break;
- case TRIO_BD:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t%s:2\n", chr,start,end,s1,s3); break;
- case TRIO_CA:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t%s:1\n", chr,start,end,s3,s1); break;
- case TRIO_DA:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t%s:1\n", chr,start,end,s3,s1); break;
- case TRIO_CB:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:1\t%s:2\n", chr,start,end,s3,s1); break;
- case TRIO_DB:
- fprintf(args->fp,"SG\t%s\t%d\t%d\t%s:2\t%s:2\n", chr,start,end,s3,s1); break;
- }
- if ( hap_switch[state][prev_state] & SW_MOTHER ) nswitch_mother++;
- if ( hap_switch[state][prev_state] & SW_FATHER ) nswitch_father++;
- }
- iprev = i-1;
- }
- prev_state = state;
- }
- float mrate = args->nhet_mother>1 ? (float)nswitch_mother/(args->nhet_mother-1) : 0;
- float frate = args->nhet_father>1 ? (float)nswitch_father/(args->nhet_father-1) : 0;
- fprintf(args->fp,"SW\t%s\t%s\t%d\t%d\t%f\n", s1,bcf_hdr_id2name(args->hdr,args->prev_rid),args->nhet_mother,nswitch_mother,mrate);
- fprintf(args->fp,"SW\t%s\t%s\t%d\t%d\t%f\n", s3,bcf_hdr_id2name(args->hdr,args->prev_rid),args->nhet_father,nswitch_father,frate);
- args->nsites = 0;
- args->nhet_father = args->nhet_mother = 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- if ( args.prev_rid==-1 ) args.prev_rid = rec->rid;
- if ( args.prev_rid!=rec->rid ) flush_viterbi(&args);
- args.prev_rid = rec->rid;
- args.set_observed_prob(rec);
- return NULL;
-}
-
-void destroy(void)
-{
- flush_viterbi(&args);
- fclose(args.fp);
-
- free(args.gt_arr);
- free(args.tprob);
- free(args.sites);
- free(args.eprob);
- hmm_destroy(args.hmm);
-}
-
-
-
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2018 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <errno.h>
-#include <unistd.h> // for isatty
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/synced_bcf_reader.h>
-#include "bcftools.h"
-#include "filter.h"
-
-
-// Logic of the filters: include or exclude sites which match the filters?
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-typedef struct
-{
- int argc, filter_logic, regions_is_file, targets_is_file, output_type;
- char **argv, *output_fname, *fname, *regions, *targets, *filter_str;
- char *bg_samples_str, *novel_samples_str;
- int *bg_smpl, *novel_smpl, nbg_smpl, nnovel_smpl;
- filter_t *filter;
- bcf_srs_t *sr;
- bcf_hdr_t *hdr, *hdr_out;
- htsFile *out_fh;
- int32_t *gts;
- int mgts;
- uint32_t *bg_gts;
- int nbg_gts, mbg_gts, ntotal, nskipped, ntested, nnovel_al, nnovel_gt;
- kstring_t novel_als_smpl, novel_gts_smpl;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Find novel alleles and genotypes in two groups of samples.\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Finds novel alleles and genotypes in two groups of samples. Adds\n"
- " an annotation which lists samples with a novel allele (INFO/NOVELAL)\n"
- " or a novel genotype (INFO/NOVELGT)\n"
- "Usage: bcftools +contrast [Plugin Options]\n"
- "Plugin options:\n"
- " -0, --bg-samples <list> list of background samples\n"
- " -1, --novel-samples <list> list of samples where novel allele or genotype are expected\n"
- " -e, --exclude EXPR exclude sites and samples for which the expression is true\n"
- " -i, --include EXPR include sites and samples for which the expression is true\n"
- " -o, --output FILE output file name [stdout]\n"
- " -O, --output-type <b|u|z|v> b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n"
- " -r, --regions REG restrict to comma-separated list of regions\n"
- " -R, --regions-file FILE restrict to regions listed in a file\n"
- " -t, --targets REG similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file FILE similar to -R but streams rather than index-jumps\n"
- "\n"
- "Example:\n"
- " # Test if any of the samples a,b is different from the samples c,d,e\n"
- " bcftools +contrast -0 c,d,e -1 a,b file.bcf\n"
- "\n";
-}
-
-static void init_data(args_t *args)
-{
- args->sr = bcf_sr_init();
- if ( args->regions )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, args->regions, args->regions_is_file)<0 ) error("Failed to read the regions: %s\n",args->regions);
- }
- if ( args->targets && bcf_sr_set_targets(args->sr, args->targets, args->targets_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->targets);
- if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr = bcf_sr_get_header(args->sr,0);
- args->hdr_out = bcf_hdr_dup(args->hdr);
- bcf_hdr_append(args->hdr_out, "##INFO=<ID=NOVELAL,Number=.,Type=String,Description=\"List of samples with novel alleles\">");
- bcf_hdr_append(args->hdr_out, "##INFO=<ID=NOVELGT,Number=.,Type=String,Description=\"List of samples with novel genotypes. Note that only samples w/o a novel allele are listed.\">");
-
- if ( args->filter_str )
- args->filter = filter_init(args->hdr, args->filter_str);
-
- int i;
- char **smpl = hts_readlist(args->bg_samples_str, 0, &args->nbg_smpl);
- args->bg_smpl = (int*) malloc(sizeof(int)*args->nbg_smpl);
- for (i=0; i<args->nbg_smpl; i++)
- {
- args->bg_smpl[i] = bcf_hdr_id2int(args->hdr, BCF_DT_SAMPLE, smpl[i]);
- if ( args->bg_smpl[i]<0 ) error("The sample not present in the VCF: \"%s\"\n", smpl[i]);
- free(smpl[i]);
- }
- free(smpl);
-
- smpl = hts_readlist(args->novel_samples_str, 0, &args->nnovel_smpl);
- args->novel_smpl = (int*) malloc(sizeof(int)*args->nnovel_smpl);
- for (i=0; i<args->nnovel_smpl; i++)
- {
- args->novel_smpl[i] = bcf_hdr_id2int(args->hdr, BCF_DT_SAMPLE, smpl[i]);
- if ( args->novel_smpl[i]<0 ) error("The sample not present in the VCF: \"%s\"\n", smpl[i]);
- free(smpl[i]);
- }
- free(smpl);
-
- args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
- bcf_hdr_write(args->out_fh, args->hdr_out);
-}
-static void destroy_data(args_t *args)
-{
- bcf_hdr_destroy(args->hdr_out);
- hts_close(args->out_fh);
- free(args->novel_als_smpl.s);
- free(args->novel_gts_smpl.s);
- free(args->gts);
- free(args->bg_gts);
- free(args->bg_smpl);
- free(args->novel_smpl);
- if ( args->filter ) filter_destroy(args->filter);
- bcf_sr_destroy(args->sr);
- free(args);
-}
-static inline int binary_search(uint32_t val, uint32_t *dat, int ndat)
-{
- int i = -1, imin = 0, imax = ndat - 1;
- while ( imin<=imax )
- {
- i = (imin+imax)/2;
- if ( dat[i] < val ) imin = i + 1;
- else if ( dat[i] > val ) imax = i - 1;
- else return 1;
- }
- return 0;
-}
-static inline void binary_insert(uint32_t val, uint32_t **dat, int *ndat, int *mdat)
-{
- int i = -1, imin = 0, imax = *ndat - 1;
- while ( imin<=imax )
- {
- i = (imin+imax)/2;
- if ( (*dat)[i] < val ) imin = i + 1;
- else if ( (*dat)[i] > val ) imax = i - 1;
- else return;
- }
- while ( i>=0 && (*dat)[i]>val ) i--;
-
- (*ndat)++;
- hts_expand(uint32_t, (*ndat), (*mdat), (*dat));
-
- if ( *ndat > 1 )
- memmove(*dat + i + 1, *dat + i, sizeof(uint32_t)*(*ndat - i - 1));
-
- (*dat)[i+1] = val;
-}
-static int process_record(args_t *args, bcf1_t *rec)
-{
- args->ntotal++;
-
- static int warned = 0;
- int ngts = bcf_get_genotypes(args->hdr, rec, &args->gts, &args->mgts);
- ngts /= rec->n_sample;
- if ( ngts>2 ) error("todo: ploidy=%d\n", ngts);
-
- args->nbg_gts = 0;
- uint32_t bg_als = 0;
- int i,j;
- for (i=0; i<args->nbg_smpl; i++)
- {
- uint32_t gt = 0;
- int32_t *ptr = args->gts + args->bg_smpl[i]*ngts;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( bcf_gt_is_missing(ptr[j]) ) continue;
- int ial = bcf_gt_allele(ptr[j]);
- if ( ial > 31 )
- {
- if ( !warned )
- {
- fprintf(stderr,"Too many alleles (>32) at %s:%d, skipping. (todo?)\n", bcf_seqname(args->hdr,rec),rec->pos+1);
- warned = 1;
- }
- args->nskipped++;
- return -1;
- }
- bg_als |= 1<<ial;
- gt |= 1<<ial;
- }
- binary_insert(gt, &args->bg_gts, &args->nbg_gts, &args->mbg_gts);
- }
- if ( !bg_als )
- {
- // all are missing
- args->nskipped++;
- return -1;
- }
-
- args->novel_als_smpl.l = 0;
- args->novel_gts_smpl.l = 0;
-
- int has_gt = 0;
- for (i=0; i<args->nnovel_smpl; i++)
- {
- int novel_al = 0;
- uint32_t gt = 0;
- int32_t *ptr = args->gts + args->novel_smpl[i]*ngts;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( bcf_gt_is_missing(ptr[j]) ) continue;
- int ial = bcf_gt_allele(ptr[j]);
- if ( ial > 31 )
- {
- if ( !warned )
- {
- fprintf(stderr,"Too many alleles (>32) at %s:%d, skipping. (todo?)\n", bcf_seqname(args->hdr,rec),rec->pos+1);
- warned = 1;
- }
- args->nskipped++;
- return -1;
- }
- if ( !(bg_als & (1<<ial)) ) novel_al = 1;
- gt |= 1<<ial;
- }
- if ( !gt ) continue;
- has_gt = 1;
-
- char *smpl = args->hdr->samples[ args->novel_smpl[i] ];
- if ( novel_al )
- {
- if ( args->novel_als_smpl.l ) kputc(',', &args->novel_als_smpl);
- kputs(smpl, &args->novel_als_smpl);
- }
- else if ( !binary_search(gt, args->bg_gts, args->nbg_gts) )
- {
- if ( args->novel_gts_smpl.l ) kputc(',', &args->novel_gts_smpl);
- kputs(smpl, &args->novel_gts_smpl);
- }
- }
- if ( !has_gt )
- {
- // all are missing
- args->nskipped++;
- return -1;
- }
- if ( args->novel_als_smpl.l )
- {
- bcf_update_info_string(args->hdr_out, rec, "NOVELAL", args->novel_als_smpl.s);
- args->nnovel_al++;
- }
- if ( args->novel_gts_smpl.l )
- {
- bcf_update_info_string(args->hdr_out, rec, "NOVELGT", args->novel_gts_smpl.s);
- args->nnovel_gt++;
- }
- args->ntested++;
- return 0;
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->output_fname = "-";
- static struct option loptions[] =
- {
- {"bg-samples",required_argument,0,'0'},
- {"novel-samples",required_argument,0,'1'},
- {"include",required_argument,0,'i'},
- {"exclude",required_argument,0,'e'},
- {"output",required_argument,NULL,'o'},
- {"output-type",required_argument,NULL,'O'},
- {"regions",1,0,'r'},
- {"regions-file",1,0,'R'},
- {"targets",1,0,'t'},
- {"targets-file",1,0,'T'},
- {NULL,0,NULL,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "O:o:i:e:r:R:t:T:0:1:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case '0': args->bg_samples_str = optarg; break;
- case '1': args->novel_samples_str = optarg; break;
- case 'e': args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 't': args->targets = optarg; break;
- case 'T': args->targets = optarg; args->targets_is_file = 1; break;
- case 'r': args->regions = optarg; break;
- case 'R': args->regions = optarg; args->regions_is_file = 1; break;
- case 'o': args->output_fname = optarg; break;
- case 'O':
- switch (optarg[0]) {
- case 'b': args->output_type = FT_BCF_GZ; break;
- case 'u': args->output_type = FT_BCF; break;
- case 'z': args->output_type = FT_VCF_GZ; break;
- case 'v': args->output_type = FT_VCF; break;
- default: error("The output type \"%s\" not recognised\n", optarg);
- };
- break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else { error("%s",usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s",usage_text());
- else args->fname = argv[optind];
-
- init_data(args);
-
- while ( bcf_sr_next_line(args->sr) )
- {
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- if ( args->filter )
- {
- int pass = filter_test(args->filter, rec, NULL);
- if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
- if ( !pass ) continue;
- }
- process_record(args, rec);
- bcf_write(args->out_fh, args->hdr_out, rec);
- }
-
- fprintf(stderr,"Total/processed/skipped/novel_allele/novel_gt:\t%d\t%d\t%d\t%d\t%d\n", args->ntotal, args->ntested, args->nskipped, args->nnovel_al, args->nnovel_gt);
- destroy_data(args);
-
- return 0;
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2018 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <errno.h>
-#include <unistd.h> // for isatty
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/synced_bcf_reader.h>
-#include "bcftools.h"
-#include "filter.h"
-
-
-// Logic of the filters: include or exclude sites which match the filters?
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-typedef struct
-{
- int argc, filter_logic, regions_is_file, targets_is_file, output_type;
- char **argv, *output_fname, *fname, *regions, *targets, *filter_str;
- char *bg_samples_str, *novel_samples_str;
- int *bg_smpl, *novel_smpl, nbg_smpl, nnovel_smpl;
- filter_t *filter;
- bcf_srs_t *sr;
- bcf_hdr_t *hdr, *hdr_out;
- htsFile *out_fh;
- int32_t *gts;
- int mgts;
- uint32_t *bg_gts;
- int nbg_gts, mbg_gts, ntotal, nskipped, ntested, nnovel_al, nnovel_gt;
- kstring_t novel_als_smpl, novel_gts_smpl;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Find novel alleles and genotypes in two groups of samples.\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Finds novel alleles and genotypes in two groups of samples. Adds\n"
- " an annotation which lists samples with a novel allele (INFO/NOVELAL)\n"
- " or a novel genotype (INFO/NOVELGT)\n"
- "Usage: bcftools +contrast [Plugin Options]\n"
- "Plugin options:\n"
- " -0, --bg-samples <list> list of background samples\n"
- " -1, --novel-samples <list> list of samples where novel allele or genotype are expected\n"
- " -e, --exclude EXPR exclude sites and samples for which the expression is true\n"
- " -i, --include EXPR include sites and samples for which the expression is true\n"
- " -o, --output FILE output file name [bcftools_stdout]\n"
- " -O, --output-type <b|u|z|v> b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n"
- " -r, --regions REG restrict to comma-separated list of regions\n"
- " -R, --regions-file FILE restrict to regions listed in a file\n"
- " -t, --targets REG similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file FILE similar to -R but streams rather than index-jumps\n"
- "\n"
- "Example:\n"
- " # Test if any of the samples a,b is different from the samples c,d,e\n"
- " bcftools +contrast -0 c,d,e -1 a,b file.bcf\n"
- "\n";
-}
-
-static void init_data(args_t *args)
-{
- args->sr = bcf_sr_init();
- if ( args->regions )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, args->regions, args->regions_is_file)<0 ) error("Failed to read the regions: %s\n",args->regions);
- }
- if ( args->targets && bcf_sr_set_targets(args->sr, args->targets, args->targets_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->targets);
- if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr = bcf_sr_get_header(args->sr,0);
- args->hdr_out = bcf_hdr_dup(args->hdr);
- bcf_hdr_append(args->hdr_out, "##INFO=<ID=NOVELAL,Number=.,Type=String,Description=\"List of samples with novel alleles\">");
- bcf_hdr_append(args->hdr_out, "##INFO=<ID=NOVELGT,Number=.,Type=String,Description=\"List of samples with novel genotypes. Note that only samples w/o a novel allele are listed.\">");
-
- if ( args->filter_str )
- args->filter = filter_init(args->hdr, args->filter_str);
-
- int i;
- char **smpl = hts_readlist(args->bg_samples_str, 0, &args->nbg_smpl);
- args->bg_smpl = (int*) malloc(sizeof(int)*args->nbg_smpl);
- for (i=0; i<args->nbg_smpl; i++)
- {
- args->bg_smpl[i] = bcf_hdr_id2int(args->hdr, BCF_DT_SAMPLE, smpl[i]);
- if ( args->bg_smpl[i]<0 ) error("The sample not present in the VCF: \"%s\"\n", smpl[i]);
- free(smpl[i]);
- }
- free(smpl);
-
- smpl = hts_readlist(args->novel_samples_str, 0, &args->nnovel_smpl);
- args->novel_smpl = (int*) malloc(sizeof(int)*args->nnovel_smpl);
- for (i=0; i<args->nnovel_smpl; i++)
- {
- args->novel_smpl[i] = bcf_hdr_id2int(args->hdr, BCF_DT_SAMPLE, smpl[i]);
- if ( args->novel_smpl[i]<0 ) error("The sample not present in the VCF: \"%s\"\n", smpl[i]);
- free(smpl[i]);
- }
- free(smpl);
-
- args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
- bcf_hdr_write(args->out_fh, args->hdr_out);
-}
-static void destroy_data(args_t *args)
-{
- bcf_hdr_destroy(args->hdr_out);
- hts_close(args->out_fh);
- free(args->novel_als_smpl.s);
- free(args->novel_gts_smpl.s);
- free(args->gts);
- free(args->bg_gts);
- free(args->bg_smpl);
- free(args->novel_smpl);
- if ( args->filter ) filter_destroy(args->filter);
- bcf_sr_destroy(args->sr);
- free(args);
-}
-static inline int binary_search(uint32_t val, uint32_t *dat, int ndat)
-{
- int i = -1, imin = 0, imax = ndat - 1;
- while ( imin<=imax )
- {
- i = (imin+imax)/2;
- if ( dat[i] < val ) imin = i + 1;
- else if ( dat[i] > val ) imax = i - 1;
- else return 1;
- }
- return 0;
-}
-static inline void binary_insert(uint32_t val, uint32_t **dat, int *ndat, int *mdat)
-{
- int i = -1, imin = 0, imax = *ndat - 1;
- while ( imin<=imax )
- {
- i = (imin+imax)/2;
- if ( (*dat)[i] < val ) imin = i + 1;
- else if ( (*dat)[i] > val ) imax = i - 1;
- else return;
- }
- while ( i>=0 && (*dat)[i]>val ) i--;
-
- (*ndat)++;
- hts_expand(uint32_t, (*ndat), (*mdat), (*dat));
-
- if ( *ndat > 1 )
- memmove(*dat + i + 1, *dat + i, sizeof(uint32_t)*(*ndat - i - 1));
-
- (*dat)[i+1] = val;
-}
-static int process_record(args_t *args, bcf1_t *rec)
-{
- args->ntotal++;
-
- static int warned = 0;
- int ngts = bcf_get_genotypes(args->hdr, rec, &args->gts, &args->mgts);
- ngts /= rec->n_sample;
- if ( ngts>2 ) error("todo: ploidy=%d\n", ngts);
-
- args->nbg_gts = 0;
- uint32_t bg_als = 0;
- int i,j;
- for (i=0; i<args->nbg_smpl; i++)
- {
- uint32_t gt = 0;
- int32_t *ptr = args->gts + args->bg_smpl[i]*ngts;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( bcf_gt_is_missing(ptr[j]) ) continue;
- int ial = bcf_gt_allele(ptr[j]);
- if ( ial > 31 )
- {
- if ( !warned )
- {
- fprintf(bcftools_stderr,"Too many alleles (>32) at %s:%d, skipping. (todo?)\n", bcf_seqname(args->hdr,rec),rec->pos+1);
- warned = 1;
- }
- args->nskipped++;
- return -1;
- }
- bg_als |= 1<<ial;
- gt |= 1<<ial;
- }
- binary_insert(gt, &args->bg_gts, &args->nbg_gts, &args->mbg_gts);
- }
- if ( !bg_als )
- {
- // all are missing
- args->nskipped++;
- return -1;
- }
-
- args->novel_als_smpl.l = 0;
- args->novel_gts_smpl.l = 0;
-
- int has_gt = 0;
- for (i=0; i<args->nnovel_smpl; i++)
- {
- int novel_al = 0;
- uint32_t gt = 0;
- int32_t *ptr = args->gts + args->novel_smpl[i]*ngts;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( bcf_gt_is_missing(ptr[j]) ) continue;
- int ial = bcf_gt_allele(ptr[j]);
- if ( ial > 31 )
- {
- if ( !warned )
- {
- fprintf(bcftools_stderr,"Too many alleles (>32) at %s:%d, skipping. (todo?)\n", bcf_seqname(args->hdr,rec),rec->pos+1);
- warned = 1;
- }
- args->nskipped++;
- return -1;
- }
- if ( !(bg_als & (1<<ial)) ) novel_al = 1;
- gt |= 1<<ial;
- }
- if ( !gt ) continue;
- has_gt = 1;
-
- char *smpl = args->hdr->samples[ args->novel_smpl[i] ];
- if ( novel_al )
- {
- if ( args->novel_als_smpl.l ) kputc(',', &args->novel_als_smpl);
- kputs(smpl, &args->novel_als_smpl);
- }
- else if ( !binary_search(gt, args->bg_gts, args->nbg_gts) )
- {
- if ( args->novel_gts_smpl.l ) kputc(',', &args->novel_gts_smpl);
- kputs(smpl, &args->novel_gts_smpl);
- }
- }
- if ( !has_gt )
- {
- // all are missing
- args->nskipped++;
- return -1;
- }
- if ( args->novel_als_smpl.l )
- {
- bcf_update_info_string(args->hdr_out, rec, "NOVELAL", args->novel_als_smpl.s);
- args->nnovel_al++;
- }
- if ( args->novel_gts_smpl.l )
- {
- bcf_update_info_string(args->hdr_out, rec, "NOVELGT", args->novel_gts_smpl.s);
- args->nnovel_gt++;
- }
- args->ntested++;
- return 0;
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->output_fname = "-";
- static struct option loptions[] =
- {
- {"bg-samples",required_argument,0,'0'},
- {"novel-samples",required_argument,0,'1'},
- {"include",required_argument,0,'i'},
- {"exclude",required_argument,0,'e'},
- {"output",required_argument,NULL,'o'},
- {"output-type",required_argument,NULL,'O'},
- {"regions",1,0,'r'},
- {"regions-file",1,0,'R'},
- {"targets",1,0,'t'},
- {"targets-file",1,0,'T'},
- {NULL,0,NULL,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "O:o:i:e:r:R:t:T:0:1:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case '0': args->bg_samples_str = optarg; break;
- case '1': args->novel_samples_str = optarg; break;
- case 'e': args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 't': args->targets = optarg; break;
- case 'T': args->targets = optarg; args->targets_is_file = 1; break;
- case 'r': args->regions = optarg; break;
- case 'R': args->regions = optarg; args->regions_is_file = 1; break;
- case 'o': args->output_fname = optarg; break;
- case 'O':
- switch (optarg[0]) {
- case 'b': args->output_type = FT_BCF_GZ; break;
- case 'u': args->output_type = FT_BCF; break;
- case 'z': args->output_type = FT_VCF_GZ; break;
- case 'v': args->output_type = FT_VCF; break;
- default: error("The output type \"%s\" not recognised\n", optarg);
- };
- break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else { error("%s",usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s",usage_text());
- else args->fname = argv[optind];
-
- init_data(args);
-
- while ( bcf_sr_next_line(args->sr) )
- {
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- if ( args->filter )
- {
- int pass = filter_test(args->filter, rec, NULL);
- if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
- if ( !pass ) continue;
- }
- process_record(args, rec);
- bcf_write(args->out_fh, args->hdr_out, rec);
- }
-
- fprintf(bcftools_stderr,"Total/processed/skipped/novel_allele/novel_gt:\t%d\t%d\t%d\t%d\t%d\n", args->ntotal, args->ntested, args->nskipped, args->nnovel_al, args->nnovel_gt);
- destroy_data(args);
-
- return 0;
-}
+++ /dev/null
-/* plugins/counts.c -- counts SNPs, Indels, and total number of sites.
-
- Copyright (C) 2013, 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-
-int nsamples, nsnps, nindels, nmnps, nothers, nsites;
-
-/*
- This short description is used to generate the output of `bcftools plugin -l`.
-*/
-const char *about(void)
-{
- return
- "A minimal plugin which counts number of samples, SNPs,\n"
- "INDELs, MNPs and total number of sites.\n";
-}
-
-/*
- Called once at startup, allows to initialize local variables.
- Return 1 to suppress VCF/BCF header from printing, 0 otherwise.
-*/
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- nsamples = bcf_hdr_nsamples(in);
- nsnps = nindels = nmnps = nothers = nsites = 0;
- return 1;
-}
-
-
-/*
- Called for each VCF record. Return rec to output the line or NULL
- to suppress output.
-*/
-bcf1_t *process(bcf1_t *rec)
-{
- int type = bcf_get_variant_types(rec);
- if ( type & VCF_SNP ) nsnps++;
- if ( type & VCF_INDEL ) nindels++;
- if ( type & VCF_MNP ) nmnps++;
- if ( type & VCF_OTHER ) nothers++;
- nsites++;
- return NULL;
-}
-
-
-/*
- Clean up.
-*/
-void destroy(void)
-{
- printf("Number of samples: %d\n", nsamples);
- printf("Number of SNPs: %d\n", nsnps);
- printf("Number of INDELs: %d\n", nindels);
- printf("Number of MNPs: %d\n", nmnps);
- printf("Number of others: %d\n", nothers);
- printf("Number of sites: %d\n", nsites);
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugins/counts.c -- counts SNPs, Indels, and total number of sites.
-
- Copyright (C) 2013, 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-
-int nsamples, nsnps, nindels, nmnps, nothers, nsites;
-
-/*
- This short description is used to generate the output of `bcftools plugin -l`.
-*/
-const char *about(void)
-{
- return
- "A minimal plugin which counts number of samples, SNPs,\n"
- "INDELs, MNPs and total number of sites.\n";
-}
-
-/*
- Called once at startup, allows to initialize local variables.
- Return 1 to suppress VCF/BCF header from printing, 0 otherwise.
-*/
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- nsamples = bcf_hdr_nsamples(in);
- nsnps = nindels = nmnps = nothers = nsites = 0;
- return 1;
-}
-
-
-/*
- Called for each VCF record. Return rec to output the line or NULL
- to suppress output.
-*/
-bcf1_t *process(bcf1_t *rec)
-{
- int type = bcf_get_variant_types(rec);
- if ( type & VCF_SNP ) nsnps++;
- if ( type & VCF_INDEL ) nindels++;
- if ( type & VCF_MNP ) nmnps++;
- if ( type & VCF_OTHER ) nothers++;
- nsites++;
- return NULL;
-}
-
-
-/*
- Clean up.
-*/
-void destroy(void)
-{
- fprintf(bcftools_stdout, "Number of samples: %d\n", nsamples);
- fprintf(bcftools_stdout, "Number of SNPs: %d\n", nsnps);
- fprintf(bcftools_stdout, "Number of INDELs: %d\n", nindels);
- fprintf(bcftools_stdout, "Number of MNPs: %d\n", nmnps);
- fprintf(bcftools_stdout, "Number of others: %d\n", nothers);
- fprintf(bcftools_stdout, "Number of sites: %d\n", nsites);
-}
-
-
+++ /dev/null
-/* plugins/dosage.c -- prints genotype dosage.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-#include <math.h>
-#include <getopt.h>
-#include "bcftools.h"
-
-
-/*
- This short description is used to generate the output of `bcftools plugin -l`.
-*/
-const char *about(void)
-{
- return "Prints genotype dosage determined from tags requested by the user.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Print genotype dosage\n"
- "Usage: bcftools +dosage [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -t, --tags <list> VCF tags to determine the dosage from [PL,GL,GT]\n"
- "\n"
- "Example:\n"
- " bcftools +dosage in.vcf -- -t GT\n"
- "\n";
-}
-
-bcf_hdr_t *in_hdr = NULL;
-int pl_type = 0, gl_type = 0;
-uint8_t *buf = NULL;
-int nbuf = 0; // NB: number of elements, not bytes
-char **tags = NULL;
-int ntags = 0;
-float *vals = NULL, *dsg = NULL;
-int mvals, mdsg;
-
-typedef int (*dosage_f) (bcf1_t *);
-dosage_f *handlers = NULL;
-int nhandlers = 0;
-
-
-int calc_dosage_PL(bcf1_t *rec)
-{
- int i, j, nret = bcf_get_format_values(in_hdr,rec,"PL",(void**)&buf,&nbuf,pl_type);
- if ( nret<0 ) return -1;
-
- nret /= rec->n_sample;
- if ( nret != rec->n_allele*(rec->n_allele+1)/2 ) return -1; // not diploid
- hts_expand(float, nret, mvals, vals);
- hts_expand(float, rec->n_allele, mdsg, dsg);
- #define BRANCH(type_t,is_missing,is_vector_end) \
- { \
- type_t *ptr = (type_t*) buf; \
- for (i=0; i<rec->n_sample; i++) \
- { \
- float sum = 0; \
- for (j=0; j<nret; j++) \
- { \
- if ( is_missing || is_vector_end ) break; \
- vals[j] = exp(-0.1*ptr[j]); \
- sum += vals[j]; \
- } \
- if ( j<nret ) \
- for (j=0; j<rec->n_allele; j++) dsg[j] = -1; \
- else \
- { \
- if ( sum ) for (j=0; j<nret; j++) vals[j] /= sum; \
- memset(dsg, 0, sizeof(float)*rec->n_allele); \
- int k, l = 0; \
- for (j=0; j<rec->n_allele; j++) \
- { \
- for (k=0; k<=j; k++) \
- { \
- dsg[j] += vals[l]; \
- dsg[k] += vals[l]; \
- } \
- } \
- } \
- for (j=1; j<rec->n_allele; j++) \
- printf("%c%.1f",j==1?'\t':',',dsg[j]); \
- ptr += nret; \
- } \
- }
- switch (pl_type)
- {
- case BCF_HT_INT: BRANCH(int32_t,ptr[j]==bcf_int32_missing,ptr[j]==bcf_int32_vector_end); break;
- case BCF_HT_REAL: BRANCH(float,bcf_float_is_missing(ptr[j]),bcf_float_is_vector_end(ptr[j])); break;
- }
- #undef BRANCH
- return 0;
-}
-
-int calc_dosage_GL(bcf1_t *rec)
-{
- int i, j, nret = bcf_get_format_values(in_hdr,rec,"GL",(void**)&buf,&nbuf,pl_type);
- if ( nret<0 ) return -1;
-
- nret /= rec->n_sample;
- if ( nret != rec->n_allele*(rec->n_allele+1)/2 ) return -1; // not diploid
- hts_expand(float, nret, mvals, vals);
- hts_expand(float, rec->n_allele, mdsg, dsg);
- #define BRANCH(type_t,is_missing,is_vector_end) \
- { \
- type_t *ptr = (type_t*) buf; \
- for (i=0; i<rec->n_sample; i++) \
- { \
- float sum = 0; \
- for (j=0; j<nret; j++) \
- { \
- if ( is_missing || is_vector_end ) break; \
- vals[j] = exp(ptr[j]); \
- sum += vals[j]; \
- } \
- if ( j<nret ) \
- for (j=0; j<rec->n_allele; j++) dsg[j] = -1; \
- else \
- { \
- for (; j<nret; j++) vals[j] = 0; \
- if ( sum ) for (j=0; j<nret; j++) vals[j] /= sum; \
- memset(dsg, 0, sizeof(float)*rec->n_allele); \
- int k, l = 0; \
- for (j=0; j<rec->n_allele; j++) \
- { \
- for (k=0; k<=j; k++) \
- { \
- dsg[j] += vals[l]; \
- dsg[k] += vals[l]; \
- } \
- } \
- } \
- for (j=1; j<rec->n_allele; j++) \
- printf("%c%.1f",j==1?'\t':',',dsg[j]); \
- ptr += nret; \
- } \
- }
- switch (pl_type)
- {
- case BCF_HT_INT: BRANCH(int32_t,ptr[j]==bcf_int32_missing,ptr[j]==bcf_int32_vector_end); break;
- case BCF_HT_REAL: BRANCH(float,bcf_float_is_missing(ptr[j]),bcf_float_is_vector_end(ptr[j])); break;
- }
- #undef BRANCH
- return 0;
-}
-
-int calc_dosage_GT(bcf1_t *rec)
-{
- int i, j, nret = bcf_get_genotypes(in_hdr,rec,(void**)&buf,&nbuf);
- if ( nret<0 ) return -1;
-
- nret /= rec->n_sample;
- int32_t *ptr = (int32_t*) buf;
- hts_expand(float, rec->n_allele, mdsg, dsg);
- for (i=0; i<rec->n_sample; i++)
- {
- memset(dsg, 0, sizeof(float)*rec->n_allele);
- for (j=0; j<nret; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end || bcf_gt_is_missing(ptr[j]) ) break;
- int idx = bcf_gt_allele(ptr[j]);
- if ( idx > rec->n_allele ) error("The allele index is out of range at %s:%d\n", bcf_seqname(in_hdr,rec),rec->pos+1);
- dsg[idx] += 1;
- }
- if ( !j )
- for (j=0; j<rec->n_allele; j++) dsg[j] = -1;
- for (j=1; j<rec->n_allele; j++)
- printf("%c%.1f",j==1?'\t':',',dsg[j]);
- ptr += nret;
- }
- return 0;
-}
-
-
-char **split_list(char *str, int *nitems)
-{
- int n = 0, done = 0;
- char *ss = strdup(str), **out = NULL;
- while ( !done && *ss )
- {
- char *se = ss;
- while ( *se && *se!=',' ) se++;
- if ( !*se ) done = 1;
- *se = 0;
- n++;
- out = (char**) realloc(out,sizeof(char*)*n);
- out[n-1] = ss;
- ss = se+1;
- }
- *nitems = n;
- return out;
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- int i, id, c;
- char *tags_str = "PL,GL,GT";
-
- static struct option loptions[] =
- {
- {"tags",1,0,'t'},
- {0,0,0,0}
- };
- while ((c = getopt_long(argc, argv, "t:?h",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 't': tags_str = optarg; break;
- case 'h':
- case '?':
- default: fprintf(stderr,"%s", usage()); exit(1); break;
- }
- }
- tags = split_list(tags_str, &ntags);
-
- in_hdr = in;
- for (i=0; i<ntags; i++)
- {
- if ( !strcmp("PL",tags[i]) )
- {
- id = bcf_hdr_id2int(in_hdr,BCF_DT_ID,"PL");
- if ( bcf_hdr_idinfo_exists(in_hdr,BCF_HL_FMT,id) )
- {
- pl_type = bcf_hdr_id2type(in_hdr,BCF_HL_FMT,id);
- if ( pl_type!=BCF_HT_INT && pl_type!=BCF_HT_REAL )
- {
- fprintf(stderr,"Expected numeric type of FORMAT/PL\n");
- return -1;
- }
- handlers = (dosage_f*) realloc(handlers,(nhandlers+1)*sizeof(*handlers));
- handlers[nhandlers++] = calc_dosage_PL;
- }
- }
- else if ( !strcmp("GL",tags[i]) )
- {
- id = bcf_hdr_id2int(in_hdr,BCF_DT_ID,"GL");
- if ( bcf_hdr_idinfo_exists(in_hdr,BCF_HL_FMT,id) )
- {
- gl_type = bcf_hdr_id2type(in_hdr,BCF_HL_FMT,id);
- if ( gl_type!=BCF_HT_INT && gl_type!=BCF_HT_REAL )
- {
- fprintf(stderr,"Expected numeric type of FORMAT/GL\n");
- return -1;
- }
- handlers = (dosage_f*) realloc(handlers,(nhandlers+1)*sizeof(*handlers));
- handlers[nhandlers++] = calc_dosage_GL;
- }
- }
- else if ( !strcmp("GT",tags[i]) )
- {
- handlers = (dosage_f*) realloc(handlers,(nhandlers+1)*sizeof(*handlers));
- handlers[nhandlers++] = calc_dosage_GT;
- }
- else
- {
- fprintf(stderr,"No handler for tag \"%s\"\n", tags[i]);
- return -1;
- }
- }
- free(tags[0]);
- free(tags);
-
- printf("#[1]CHROM\t[2]POS\t[3]REF\t[4]ALT");
- for (i=0; i<bcf_hdr_nsamples(in_hdr); i++) printf("\t[%d]%s", i+5,in_hdr->samples[i]);
- printf("\n");
-
- return 1;
-}
-
-
-bcf1_t *process(bcf1_t *rec)
-{
- int i,j, ret;
-
- printf("%s\t%d\t%s", bcf_seqname(in_hdr,rec),rec->pos+1,rec->d.allele[0]);
- if ( rec->n_allele == 1 ) printf("\t.");
- else for (i=1; i<rec->n_allele; i++) printf("%c%s", i==1?'\t':',', rec->d.allele[i]);
- if ( rec->n_allele==1 )
- {
- for (j=0; j<rec->n_sample; j++) printf("\t0.0");
- }
- else
- {
- for (i=0; i<nhandlers; i++)
- {
- ret = handlers[i](rec);
- if ( !ret ) break; // successfully printed
- }
- if ( i==nhandlers )
- {
- // none of the annotations present
- for (i=0; i<rec->n_sample; i++) printf("\t-1.0");
- }
- }
- printf("\n");
-
- return NULL;
-}
-
-
-void destroy(void)
-{
- free(vals);
- free(dsg);
- free(handlers);
- free(buf);
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugins/dosage.c -- prints genotype dosage.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-#include <math.h>
-#include <getopt.h>
-#include "bcftools.h"
-
-
-/*
- This short description is used to generate the output of `bcftools plugin -l`.
-*/
-const char *about(void)
-{
- return "Prints genotype dosage determined from tags requested by the user.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Print genotype dosage\n"
- "Usage: bcftools +dosage [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -t, --tags <list> VCF tags to determine the dosage from [PL,GL,GT]\n"
- "\n"
- "Example:\n"
- " bcftools +dosage in.vcf -- -t GT\n"
- "\n";
-}
-
-bcf_hdr_t *in_hdr = NULL;
-int pl_type = 0, gl_type = 0;
-uint8_t *buf = NULL;
-int nbuf = 0; // NB: number of elements, not bytes
-char **tags = NULL;
-int ntags = 0;
-float *vals = NULL, *dsg = NULL;
-int mvals, mdsg;
-
-typedef int (*dosage_f) (bcf1_t *);
-dosage_f *handlers = NULL;
-int nhandlers = 0;
-
-
-int calc_dosage_PL(bcf1_t *rec)
-{
- int i, j, nret = bcf_get_format_values(in_hdr,rec,"PL",(void**)&buf,&nbuf,pl_type);
- if ( nret<0 ) return -1;
-
- nret /= rec->n_sample;
- if ( nret != rec->n_allele*(rec->n_allele+1)/2 ) return -1; // not diploid
- hts_expand(float, nret, mvals, vals);
- hts_expand(float, rec->n_allele, mdsg, dsg);
- #define BRANCH(type_t,is_missing,is_vector_end) \
- { \
- type_t *ptr = (type_t*) buf; \
- for (i=0; i<rec->n_sample; i++) \
- { \
- float sum = 0; \
- for (j=0; j<nret; j++) \
- { \
- if ( is_missing || is_vector_end ) break; \
- vals[j] = exp(-0.1*ptr[j]); \
- sum += vals[j]; \
- } \
- if ( j<nret ) \
- for (j=0; j<rec->n_allele; j++) dsg[j] = -1; \
- else \
- { \
- if ( sum ) for (j=0; j<nret; j++) vals[j] /= sum; \
- memset(dsg, 0, sizeof(float)*rec->n_allele); \
- int k, l = 0; \
- for (j=0; j<rec->n_allele; j++) \
- { \
- for (k=0; k<=j; k++) \
- { \
- dsg[j] += vals[l]; \
- dsg[k] += vals[l]; \
- } \
- } \
- } \
- for (j=1; j<rec->n_allele; j++) \
- fprintf(bcftools_stdout, "%c%.1f",j==1?'\t':',',dsg[j]); \
- ptr += nret; \
- } \
- }
- switch (pl_type)
- {
- case BCF_HT_INT: BRANCH(int32_t,ptr[j]==bcf_int32_missing,ptr[j]==bcf_int32_vector_end); break;
- case BCF_HT_REAL: BRANCH(float,bcf_float_is_missing(ptr[j]),bcf_float_is_vector_end(ptr[j])); break;
- }
- #undef BRANCH
- return 0;
-}
-
-int calc_dosage_GL(bcf1_t *rec)
-{
- int i, j, nret = bcf_get_format_values(in_hdr,rec,"GL",(void**)&buf,&nbuf,pl_type);
- if ( nret<0 ) return -1;
-
- nret /= rec->n_sample;
- if ( nret != rec->n_allele*(rec->n_allele+1)/2 ) return -1; // not diploid
- hts_expand(float, nret, mvals, vals);
- hts_expand(float, rec->n_allele, mdsg, dsg);
- #define BRANCH(type_t,is_missing,is_vector_end) \
- { \
- type_t *ptr = (type_t*) buf; \
- for (i=0; i<rec->n_sample; i++) \
- { \
- float sum = 0; \
- for (j=0; j<nret; j++) \
- { \
- if ( is_missing || is_vector_end ) break; \
- vals[j] = exp(ptr[j]); \
- sum += vals[j]; \
- } \
- if ( j<nret ) \
- for (j=0; j<rec->n_allele; j++) dsg[j] = -1; \
- else \
- { \
- for (; j<nret; j++) vals[j] = 0; \
- if ( sum ) for (j=0; j<nret; j++) vals[j] /= sum; \
- memset(dsg, 0, sizeof(float)*rec->n_allele); \
- int k, l = 0; \
- for (j=0; j<rec->n_allele; j++) \
- { \
- for (k=0; k<=j; k++) \
- { \
- dsg[j] += vals[l]; \
- dsg[k] += vals[l]; \
- } \
- } \
- } \
- for (j=1; j<rec->n_allele; j++) \
- fprintf(bcftools_stdout, "%c%.1f",j==1?'\t':',',dsg[j]); \
- ptr += nret; \
- } \
- }
- switch (pl_type)
- {
- case BCF_HT_INT: BRANCH(int32_t,ptr[j]==bcf_int32_missing,ptr[j]==bcf_int32_vector_end); break;
- case BCF_HT_REAL: BRANCH(float,bcf_float_is_missing(ptr[j]),bcf_float_is_vector_end(ptr[j])); break;
- }
- #undef BRANCH
- return 0;
-}
-
-int calc_dosage_GT(bcf1_t *rec)
-{
- int i, j, nret = bcf_get_genotypes(in_hdr,rec,(void**)&buf,&nbuf);
- if ( nret<0 ) return -1;
-
- nret /= rec->n_sample;
- int32_t *ptr = (int32_t*) buf;
- hts_expand(float, rec->n_allele, mdsg, dsg);
- for (i=0; i<rec->n_sample; i++)
- {
- memset(dsg, 0, sizeof(float)*rec->n_allele);
- for (j=0; j<nret; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end || bcf_gt_is_missing(ptr[j]) ) break;
- int idx = bcf_gt_allele(ptr[j]);
- if ( idx > rec->n_allele ) error("The allele index is out of range at %s:%d\n", bcf_seqname(in_hdr,rec),rec->pos+1);
- dsg[idx] += 1;
- }
- if ( !j )
- for (j=0; j<rec->n_allele; j++) dsg[j] = -1;
- for (j=1; j<rec->n_allele; j++)
- fprintf(bcftools_stdout, "%c%.1f",j==1?'\t':',',dsg[j]);
- ptr += nret;
- }
- return 0;
-}
-
-
-char **split_list(char *str, int *nitems)
-{
- int n = 0, done = 0;
- char *ss = strdup(str), **out = NULL;
- while ( !done && *ss )
- {
- char *se = ss;
- while ( *se && *se!=',' ) se++;
- if ( !*se ) done = 1;
- *se = 0;
- n++;
- out = (char**) realloc(out,sizeof(char*)*n);
- out[n-1] = ss;
- ss = se+1;
- }
- *nitems = n;
- return out;
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- int i, id, c;
- char *tags_str = "PL,GL,GT";
-
- static struct option loptions[] =
- {
- {"tags",1,0,'t'},
- {0,0,0,0}
- };
- while ((c = getopt_long(argc, argv, "t:?h",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 't': tags_str = optarg; break;
- case 'h':
- case '?':
- default: fprintf(bcftools_stderr,"%s", usage()); exit(1); break;
- }
- }
- tags = split_list(tags_str, &ntags);
-
- in_hdr = in;
- for (i=0; i<ntags; i++)
- {
- if ( !strcmp("PL",tags[i]) )
- {
- id = bcf_hdr_id2int(in_hdr,BCF_DT_ID,"PL");
- if ( bcf_hdr_idinfo_exists(in_hdr,BCF_HL_FMT,id) )
- {
- pl_type = bcf_hdr_id2type(in_hdr,BCF_HL_FMT,id);
- if ( pl_type!=BCF_HT_INT && pl_type!=BCF_HT_REAL )
- {
- fprintf(bcftools_stderr,"Expected numeric type of FORMAT/PL\n");
- return -1;
- }
- handlers = (dosage_f*) realloc(handlers,(nhandlers+1)*sizeof(*handlers));
- handlers[nhandlers++] = calc_dosage_PL;
- }
- }
- else if ( !strcmp("GL",tags[i]) )
- {
- id = bcf_hdr_id2int(in_hdr,BCF_DT_ID,"GL");
- if ( bcf_hdr_idinfo_exists(in_hdr,BCF_HL_FMT,id) )
- {
- gl_type = bcf_hdr_id2type(in_hdr,BCF_HL_FMT,id);
- if ( gl_type!=BCF_HT_INT && gl_type!=BCF_HT_REAL )
- {
- fprintf(bcftools_stderr,"Expected numeric type of FORMAT/GL\n");
- return -1;
- }
- handlers = (dosage_f*) realloc(handlers,(nhandlers+1)*sizeof(*handlers));
- handlers[nhandlers++] = calc_dosage_GL;
- }
- }
- else if ( !strcmp("GT",tags[i]) )
- {
- handlers = (dosage_f*) realloc(handlers,(nhandlers+1)*sizeof(*handlers));
- handlers[nhandlers++] = calc_dosage_GT;
- }
- else
- {
- fprintf(bcftools_stderr,"No handler for tag \"%s\"\n", tags[i]);
- return -1;
- }
- }
- free(tags[0]);
- free(tags);
-
- fprintf(bcftools_stdout, "#[1]CHROM\t[2]POS\t[3]REF\t[4]ALT");
- for (i=0; i<bcf_hdr_nsamples(in_hdr); i++) fprintf(bcftools_stdout, "\t[%d]%s", i+5,in_hdr->samples[i]);
- fprintf(bcftools_stdout, "\n");
-
- return 1;
-}
-
-
-bcf1_t *process(bcf1_t *rec)
-{
- int i,j, ret;
-
- fprintf(bcftools_stdout, "%s\t%d\t%s", bcf_seqname(in_hdr,rec),rec->pos+1,rec->d.allele[0]);
- if ( rec->n_allele == 1 ) fprintf(bcftools_stdout, "\t.");
- else for (i=1; i<rec->n_allele; i++) fprintf(bcftools_stdout, "%c%s", i==1?'\t':',', rec->d.allele[i]);
- if ( rec->n_allele==1 )
- {
- for (j=0; j<rec->n_sample; j++) fprintf(bcftools_stdout, "\t0.0");
- }
- else
- {
- for (i=0; i<nhandlers; i++)
- {
- ret = handlers[i](rec);
- if ( !ret ) break; // successfully printed
- }
- if ( i==nhandlers )
- {
- // none of the annotations present
- for (i=0; i<rec->n_sample; i++) fprintf(bcftools_stdout, "\t-1.0");
- }
- }
- fprintf(bcftools_stdout, "\n");
-
- return NULL;
-}
-
-
-void destroy(void)
-{
- free(vals);
- free(dsg);
- free(handlers);
- free(buf);
-}
-
-
+++ /dev/null
-/* plugins/fill-AN-AC.c -- fills AN and AC INFO fields.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/vcfutils.h>
-
-bcf_hdr_t *in_hdr, *out_hdr;
-int *arr = NULL, marr = 0;
-
-const char *about(void)
-{
- return "Fill INFO fields AN and AC.\n";
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- in_hdr = in;
- out_hdr = out;
- bcf_hdr_append(out_hdr, "##INFO=<ID=AC,Number=A,Type=Integer,Description=\"Allele count in genotypes\">");
- bcf_hdr_append(out_hdr, "##INFO=<ID=AN,Number=1,Type=Integer,Description=\"Total number of alleles in called genotypes\">");
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- hts_expand(int,rec->n_allele,marr,arr);
- int ret = bcf_calc_ac(in_hdr,rec,arr,BCF_UN_FMT);
- if ( ret>0 )
- {
- int i, an = 0;
- for (i=0; i<rec->n_allele; i++) an += arr[i];
- bcf_update_info_int32(out_hdr, rec, "AN", &an, 1);
- bcf_update_info_int32(out_hdr, rec, "AC", arr+1, rec->n_allele-1);
- }
- return rec;
-}
-
-void destroy(void)
-{
- free(arr);
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugins/fill-AN-AC.c -- fills AN and AC INFO fields.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/vcfutils.h>
-
-bcf_hdr_t *in_hdr, *out_hdr;
-int *arr = NULL, marr = 0;
-
-const char *about(void)
-{
- return "Fill INFO fields AN and AC.\n";
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- in_hdr = in;
- out_hdr = out;
- bcf_hdr_append(out_hdr, "##INFO=<ID=AC,Number=A,Type=Integer,Description=\"Allele count in genotypes\">");
- bcf_hdr_append(out_hdr, "##INFO=<ID=AN,Number=1,Type=Integer,Description=\"Total number of alleles in called genotypes\">");
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- hts_expand(int,rec->n_allele,marr,arr);
- int ret = bcf_calc_ac(in_hdr,rec,arr,BCF_UN_FMT);
- if ( ret>0 )
- {
- int i, an = 0;
- for (i=0; i<rec->n_allele; i++) an += arr[i];
- bcf_update_info_int32(out_hdr, rec, "AN", &an, 1);
- bcf_update_info_int32(out_hdr, rec, "AC", arr+1, rec->n_allele-1);
- }
- return rec;
-}
-
-void destroy(void)
-{
- free(arr);
-}
-
-
+++ /dev/null
-/* plugin/fill-from-fasta.c -- fill-from-fasta plugin.
-
- Copyright (C) 2016 Genome Research Ltd.
-
- Author: Shane McCarthy <sm15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <htslib/vcf.h>
-#include <htslib/faidx.h>
-#include <htslib/kseq.h>
-#include "filter.h"
-#include "bcftools.h"
-
-const char *about(void)
-{
- return "Fill INFO or REF field based on values in a fasta file\n";
-}
-
-const char *usage(void)
-{
- return
-"\n"
-"About: Fill INFO or REF field based on values in a fasta file.\n"
-" The fasta file must be indexed with samtools faidx.\n"
-"Usage: bcftools +fill-from-fasta [General Options] -- [Plugin Options]\n"
-"\n"
-"General options:\n"
-" run \"bcftools plugin\" for a list of common options\n"
-"\n"
-"Plugin options:\n"
-" -c, --column <str> REF or INFO tag, e.g. AA for ancestral allele\n"
-" -f, --fasta <file> fasta file\n"
-" -h, --header-lines <file> optional file containing header lines to append\n"
-" -i, --include <expr> annotate only records passing filter expression\n"
-" -e, --exclude <expr> annotate only records failing filter expression\n"
-
-"\n"
-"Examples:\n"
-" # fill ancestral allele as INFO/AA for SNP records\n"
-" echo '##INFO=<ID=AA,Number=1,Type=String,Description=\"Ancestral allele\">' > aa.hdr\n"
-" bcftools +fill-from-fasta in.vcf -- -c AA -f aa.fasta -h aa.hdr -i 'TYPE=\"snp\"'\n"
-"\n"
-" # fix the REF allele in VCFs where REF=N or other\n"
-" bcftools +fill-from-fasta in.vcf -- -c REF -f reference.fasta\n"
-"\n"
-" # select sites marked as P (PASS) in the 1000G Phase3 mask\n"
-" echo '##INFO=<ID=P3_MASK,Number=1,Type=String,Description=\"1000G Phase 3 mask\">' > mask.hdr\n"
-" bcftools +fill-from-fasta in.vcf -Ou -- -c P3_MASK -f 1000G_mask.fasta -h mask.hdr | bcftools view -i 'P3_MASK=\"P\"'\n"
-"\n";
-}
-
-bcf_hdr_t *in_hdr = NULL, *out_hdr = NULL;
-faidx_t *faidx;
-int anno = 0;
-char *column = NULL;
-
-#define ANNO_REF 1
-#define ANNO_STRING 2
-#define ANNO_INT 3
-
-filter_t *filter;
-char *filter_str;
-int filter_logic; // one of FLT_INCLUDE/FLT_EXCLUDE (-i or -e)
-
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- int c;
- char *ref_fname = NULL, *header_fname = NULL;
- static struct option loptions[] =
- {
- {"exclude",required_argument,NULL,'e'},
- {"include",required_argument,NULL,'i'},
- {"column",required_argument,NULL,'c'},
- {"fasta",required_argument,NULL,'f'},
- {"header-lines",required_argument,NULL,'h'},
- {NULL,0,NULL,0}
- };
- while ((c = getopt_long(argc, argv, "c:f:?h:i:e:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'e': filter_str = optarg; filter_logic |= FLT_EXCLUDE; break;
- case 'i': filter_str = optarg; filter_logic |= FLT_INCLUDE; break;
- case 'c': column = optarg; break;
- case 'f': ref_fname = optarg; break;
- case 'h': header_fname = optarg; break;
- case '?':
- default: fprintf(stderr,"%s", usage()); exit(1); break;
- }
- }
- in_hdr = in;
- out_hdr = out;
- if ( filter_logic == (FLT_EXCLUDE|FLT_INCLUDE) ) { fprintf(stderr,"Only one of -i or -e can be given.\n"); return -1; }
-
- if ( !column )
- {
- fprintf(stderr,"--column option is required.\n");
- return -1;
- }
- if (header_fname)
- {
- htsFile *file = hts_open(header_fname, "rb");
- if ( !file ) { fprintf(stderr,"Error reading %s\n", header_fname); return -1; }
- kstring_t str = {0,0,0};
- while ( hts_getline(file, KS_SEP_LINE, &str) > 0 )
- {
- if ( bcf_hdr_append(out_hdr, str.s) ) { fprintf(stderr,"Could not parse %s: %s\n", header_fname, str.s); return -1; }
- }
- hts_close(file);
- free(str.s);
- bcf_hdr_sync(out_hdr);
- }
- if (!strcasecmp("REF", column)) anno = ANNO_REF;
- else {
- if ( !strncasecmp(column,"INFO/",5) ) column += 5;
- int hdr_id = bcf_hdr_id2int(out_hdr, BCF_DT_ID, column);
- if (hdr_id<0) { fprintf(stderr,"No header ID found for %s. Header lines can be added with the --header-lines option\n", column); return -1; }
- switch ( bcf_hdr_id2type(out_hdr,BCF_HL_INFO,hdr_id) )
- {
- case BCF_HT_INT:
- anno=ANNO_INT;
- break;
- case BCF_HT_STR:
- anno=ANNO_STRING;
- break;
- default:
- fprintf(stderr,"The type of %s not recognised (%d)\n", column, bcf_hdr_id2type(out_hdr,BCF_HL_INFO,hdr_id));
- return -1;
- }
- }
- if ( !ref_fname )
- {
- fprintf(stderr,"No fasta given.\n");
- return -1;
- }
- faidx = fai_load(ref_fname);
- if ( filter_str )
- filter = filter_init(in, filter_str);
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- // filter determines if we will annotate the record
- // return record unchanged if filter applied
- if ( filter )
- {
- int ret = filter_test(filter, rec, NULL);
- if ( filter_logic==FLT_INCLUDE ) { if ( !ret ) return rec; }
- else if ( ret ) return rec;
- }
-
- int i;
- char *ref = rec->d.allele[0];
- int ref_len = strlen(ref);
- int fa_len;
- // could be sped up here by fetching the whole chromosome? could assume
- // sorted, but revert to this when non-sorted records found?
- char *fa = faidx_fetch_seq(faidx, bcf_seqname(in_hdr,rec), rec->pos, rec->pos+ref_len-1, &fa_len);
- if ( !fa ) error("faidx_fetch_seq failed at %s:%d\n", bcf_hdr_id2name(in_hdr,rec->rid), rec->pos+1);
- for (i=0; i<fa_len; i++)
- if ( (int)fa[i]>96 ) fa[i] -= 32;
-
- assert(ref_len == fa_len);
- if (anno==ANNO_REF)
- strncpy(rec->d.allele[0], fa, fa_len);
- else if (anno==ANNO_STRING)
- bcf_update_info_string(out_hdr, rec, column, fa);
- else if (anno==ANNO_INT && ref_len==1)
- {
- int val = atoi(&fa[0]);
- bcf_update_info_int32(out_hdr, rec, column, &val, 1);
- }
- free(fa);
- return rec;
-}
-
-void destroy(void)
-{
- fai_destroy(faidx);
- if (filter) filter_destroy(filter);
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugin/fill-from-fasta.c -- fill-from-fasta plugin.
-
- Copyright (C) 2016 Genome Research Ltd.
-
- Author: Shane McCarthy <sm15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <htslib/vcf.h>
-#include <htslib/faidx.h>
-#include <htslib/kseq.h>
-#include "filter.h"
-#include "bcftools.h"
-
-const char *about(void)
-{
- return "Fill INFO or REF field based on values in a fasta file\n";
-}
-
-const char *usage(void)
-{
- return
-"\n"
-"About: Fill INFO or REF field based on values in a fasta file.\n"
-" The fasta file must be indexed with samtools faidx.\n"
-"Usage: bcftools +fill-from-fasta [General Options] -- [Plugin Options]\n"
-"\n"
-"General options:\n"
-" run \"bcftools plugin\" for a list of common options\n"
-"\n"
-"Plugin options:\n"
-" -c, --column <str> REF or INFO tag, e.g. AA for ancestral allele\n"
-" -f, --fasta <file> fasta file\n"
-" -h, --header-lines <file> optional file containing header lines to append\n"
-" -i, --include <expr> annotate only records passing filter expression\n"
-" -e, --exclude <expr> annotate only records failing filter expression\n"
-
-"\n"
-"Examples:\n"
-" # fill ancestral allele as INFO/AA for SNP records\n"
-" echo '##INFO=<ID=AA,Number=1,Type=String,Description=\"Ancestral allele\">' > aa.hdr\n"
-" bcftools +fill-from-fasta in.vcf -- -c AA -f aa.fasta -h aa.hdr -i 'TYPE=\"snp\"'\n"
-"\n"
-" # fix the REF allele in VCFs where REF=N or other\n"
-" bcftools +fill-from-fasta in.vcf -- -c REF -f reference.fasta\n"
-"\n"
-" # select sites marked as P (PASS) in the 1000G Phase3 mask\n"
-" echo '##INFO=<ID=P3_MASK,Number=1,Type=String,Description=\"1000G Phase 3 mask\">' > mask.hdr\n"
-" bcftools +fill-from-fasta in.vcf -Ou -- -c P3_MASK -f 1000G_mask.fasta -h mask.hdr | bcftools view -i 'P3_MASK=\"P\"'\n"
-"\n";
-}
-
-bcf_hdr_t *in_hdr = NULL, *out_hdr = NULL;
-faidx_t *faidx;
-int anno = 0;
-char *column = NULL;
-
-#define ANNO_REF 1
-#define ANNO_STRING 2
-#define ANNO_INT 3
-
-filter_t *filter;
-char *filter_str;
-int filter_logic; // one of FLT_INCLUDE/FLT_EXCLUDE (-i or -e)
-
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- int c;
- char *ref_fname = NULL, *header_fname = NULL;
- static struct option loptions[] =
- {
- {"exclude",required_argument,NULL,'e'},
- {"include",required_argument,NULL,'i'},
- {"column",required_argument,NULL,'c'},
- {"fasta",required_argument,NULL,'f'},
- {"header-lines",required_argument,NULL,'h'},
- {NULL,0,NULL,0}
- };
- while ((c = getopt_long(argc, argv, "c:f:?h:i:e:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'e': filter_str = optarg; filter_logic |= FLT_EXCLUDE; break;
- case 'i': filter_str = optarg; filter_logic |= FLT_INCLUDE; break;
- case 'c': column = optarg; break;
- case 'f': ref_fname = optarg; break;
- case 'h': header_fname = optarg; break;
- case '?':
- default: fprintf(bcftools_stderr,"%s", usage()); exit(1); break;
- }
- }
- in_hdr = in;
- out_hdr = out;
- if ( filter_logic == (FLT_EXCLUDE|FLT_INCLUDE) ) { fprintf(bcftools_stderr,"Only one of -i or -e can be given.\n"); return -1; }
-
- if ( !column )
- {
- fprintf(bcftools_stderr,"--column option is required.\n");
- return -1;
- }
- if (header_fname)
- {
- htsFile *file = hts_open(header_fname, "rb");
- if ( !file ) { fprintf(bcftools_stderr,"Error reading %s\n", header_fname); return -1; }
- kstring_t str = {0,0,0};
- while ( hts_getline(file, KS_SEP_LINE, &str) > 0 )
- {
- if ( bcf_hdr_append(out_hdr, str.s) ) { fprintf(bcftools_stderr,"Could not parse %s: %s\n", header_fname, str.s); return -1; }
- }
- hts_close(file);
- free(str.s);
- bcf_hdr_sync(out_hdr);
- }
- if (!strcasecmp("REF", column)) anno = ANNO_REF;
- else {
- if ( !strncasecmp(column,"INFO/",5) ) column += 5;
- int hdr_id = bcf_hdr_id2int(out_hdr, BCF_DT_ID, column);
- if (hdr_id<0) { fprintf(bcftools_stderr,"No header ID found for %s. Header lines can be added with the --header-lines option\n", column); return -1; }
- switch ( bcf_hdr_id2type(out_hdr,BCF_HL_INFO,hdr_id) )
- {
- case BCF_HT_INT:
- anno=ANNO_INT;
- break;
- case BCF_HT_STR:
- anno=ANNO_STRING;
- break;
- default:
- fprintf(bcftools_stderr,"The type of %s not recognised (%d)\n", column, bcf_hdr_id2type(out_hdr,BCF_HL_INFO,hdr_id));
- return -1;
- }
- }
- if ( !ref_fname )
- {
- fprintf(bcftools_stderr,"No fasta given.\n");
- return -1;
- }
- faidx = fai_load(ref_fname);
- if ( filter_str )
- filter = filter_init(in, filter_str);
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- // filter determines if we will annotate the record
- // return record unchanged if filter applied
- if ( filter )
- {
- int ret = filter_test(filter, rec, NULL);
- if ( filter_logic==FLT_INCLUDE ) { if ( !ret ) return rec; }
- else if ( ret ) return rec;
- }
-
- int i;
- char *ref = rec->d.allele[0];
- int ref_len = strlen(ref);
- int fa_len;
- // could be sped up here by fetching the whole chromosome? could assume
- // sorted, but revert to this when non-sorted records found?
- char *fa = faidx_fetch_seq(faidx, bcf_seqname(in_hdr,rec), rec->pos, rec->pos+ref_len-1, &fa_len);
- if ( !fa ) error("faidx_fetch_seq failed at %s:%d\n", bcf_hdr_id2name(in_hdr,rec->rid), rec->pos+1);
- for (i=0; i<fa_len; i++)
- if ( (int)fa[i]>96 ) fa[i] -= 32;
-
- assert(ref_len == fa_len);
- if (anno==ANNO_REF)
- strncpy(rec->d.allele[0], fa, fa_len);
- else if (anno==ANNO_STRING)
- bcf_update_info_string(out_hdr, rec, column, fa);
- else if (anno==ANNO_INT && ref_len==1)
- {
- int val = atoi(&fa[0]);
- bcf_update_info_int32(out_hdr, rec, column, &val, 1);
- }
- free(fa);
- return rec;
-}
-
-void destroy(void)
-{
- fai_destroy(faidx);
- if (filter) filter_destroy(filter);
-}
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2015 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/kseq.h>
-#include <htslib/vcf.h>
-#include <htslib/khash_str2int.h>
-#include "bcftools.h"
-
-#define SET_AN (1<<0)
-#define SET_AC (1<<1)
-#define SET_AC_Hom (1<<2)
-#define SET_AC_Het (1<<3)
-#define SET_AC_Hemi (1<<4)
-#define SET_AF (1<<5)
-#define SET_NS (1<<6)
-#define SET_MAF (1<<7)
-#define SET_HWE (1<<8)
-#define SET_ExcHet (1<<9)
-
-typedef struct
-{
- int nhom, nhet, nhemi, nac;
-}
-counts_t;
-
-typedef struct
-{
- int ns;
- int ncounts, mcounts;
- counts_t *counts;
- char *name, *suffix;
- int nsmpl, *smpl;
-}
-pop_t;
-
-typedef struct
-{
- bcf_hdr_t *in_hdr, *out_hdr;
- int npop, tags, drop_missing, gt_id;
- pop_t *pop, **smpl2pop;
- float *farr;
- int32_t *iarr, niarr, miarr, nfarr, mfarr;
- double *hwe_probs;
- int mhwe_probs;
- kstring_t str;
-}
-args_t;
-
-static args_t *args;
-
-const char *about(void)
-{
- return "Set INFO tags AF, AC, AC_Hemi, AC_Hom, AC_Het, AN, ExcHet, HWE, MAF, NS.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Set INFO tags AF, AC, AC_Hemi, AC_Hom, AC_Het, AN, ExcHet, HWE, MAF, NS.\n"
- "Usage: bcftools +fill-tags [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -d, --drop-missing do not count half-missing genotypes \"./1\" as hemizygous\n"
- " -l, --list-tags list available tags with description\n"
- " -t, --tags LIST list of output tags. By default, all tags are filled.\n"
- " -S, --samples-file FILE list of samples (first column) and comma-separated list of populations (second column)\n"
- "\n"
- "Example:\n"
- " bcftools +fill-tags in.bcf -Ob -o out.bcf\n"
- " bcftools +fill-tags in.bcf -Ob -o out.bcf -- -t AN,AC\n"
- " bcftools +fill-tags in.bcf -Ob -o out.bcf -- -d\n"
- " bcftools +fill-tags in.bcf -Ob -o out.bcf -- -S sample-group.txt -t HWE\n"
- "\n";
-}
-
-void parse_samples(args_t *args, char *fname)
-{
- htsFile *fp = hts_open(fname, "r");
- if ( !fp ) error("Could not read: %s\n", fname);
-
- void *pop2i = khash_str2int_init();
- void *smpli = khash_str2int_init();
- kstring_t str = {0,0,0};
-
- int moff = 0, *off = NULL, nsmpl = 0;
- while ( hts_getline(fp, KS_SEP_LINE, &str)>=0 )
- {
- // NA12400 GRP1
- // NA18507 GRP1,GRP2
- char *pop_names = str.s + str.l - 1;
- while ( pop_names >= str.s && isspace(*pop_names) ) pop_names--;
- if ( pop_names <= str.s ) error("Could not parse the file: %s\n", str.s);
- pop_names[1] = 0; // trailing spaces
- while ( pop_names >= str.s && !isspace(*pop_names) ) pop_names--;
- if ( pop_names <= str.s ) error("Could not parse the file: %s\n", str.s);
-
- char *smpl = pop_names++;
- while ( smpl >= str.s && isspace(*smpl) ) smpl--;
- if ( smpl <= str.s+1 ) error("Could not parse the file: %s\n", str.s);
- smpl[1] = 0;
- smpl = str.s;
-
- int ismpl = bcf_hdr_id2int(args->in_hdr,BCF_DT_SAMPLE,smpl);
- if ( ismpl<0 )
- {
- fprintf(stderr,"Warning: The sample not present in the VCF: %s\n",smpl);
- continue;
- }
- if ( khash_str2int_has_key(smpli,smpl) )
- {
- fprintf(stderr,"Warning: The sample is listed twice in %s: %s\n",fname,smpl);
- continue;
- }
- khash_str2int_inc(smpli,strdup(smpl));
-
- int i,npops = ksplit_core(pop_names,',',&moff,&off);
- for (i=0; i<npops; i++)
- {
- char *pop_name = &pop_names[off[i]];
- if ( !khash_str2int_has_key(pop2i,pop_name) )
- {
- pop_name = strdup(pop_name);
- khash_str2int_set(pop2i,pop_name,args->npop);
- args->npop++;
- args->pop = (pop_t*) realloc(args->pop,args->npop*sizeof(*args->pop));
- memset(args->pop+args->npop-1,0,sizeof(*args->pop));
- args->pop[args->npop-1].name = pop_name;
- args->pop[args->npop-1].suffix = (char*)malloc(strlen(pop_name)+2);
- memcpy(args->pop[args->npop-1].suffix+1,pop_name,strlen(pop_name)+1);
- args->pop[args->npop-1].suffix[0] = '_';
- }
- int ipop = 0;
- khash_str2int_get(pop2i,pop_name,&ipop);
- pop_t *pop = &args->pop[ipop];
- pop->nsmpl++;
- pop->smpl = (int*) realloc(pop->smpl,pop->nsmpl*sizeof(*pop->smpl));
- pop->smpl[pop->nsmpl-1] = ismpl;
- }
- nsmpl++;
- }
-
- if ( nsmpl != bcf_hdr_nsamples(args->in_hdr) )
- fprintf(stderr,"Warning: %d samples in the list, %d samples in the VCF.\n", nsmpl,bcf_hdr_nsamples(args->in_hdr));
-
- if ( !args->npop ) error("No populations given?\n");
-
- khash_str2int_destroy(pop2i);
- khash_str2int_destroy_free(smpli);
- free(str.s);
- free(off);
- hts_close(fp);
-}
-
-void init_pops(args_t *args)
-{
- int i,j, nsmpl;
-
- // add the population "ALL", which is a summary population for all samples
- args->npop++;
- args->pop = (pop_t*) realloc(args->pop,args->npop*sizeof(*args->pop));
- memset(args->pop+args->npop-1,0,sizeof(*args->pop));
- args->pop[args->npop-1].name = strdup("");
- args->pop[args->npop-1].suffix = strdup("");
-
- nsmpl = bcf_hdr_nsamples(args->in_hdr);
- args->smpl2pop = (pop_t**) calloc(nsmpl*(args->npop+1),sizeof(pop_t*));
- for (i=0; i<nsmpl; i++)
- args->smpl2pop[i*(args->npop+1)] = &args->pop[args->npop-1];
-
- for (i=0; i<args->npop; i++)
- {
- for (j=0; j<args->pop[i].nsmpl; j++)
- {
- int ismpl = args->pop[i].smpl[j];
- pop_t **smpl2pop = &args->smpl2pop[ismpl*(args->npop+1)];
- while (*smpl2pop) smpl2pop++;
- *smpl2pop = &args->pop[i];
- }
- }
-}
-
-int parse_tags(args_t *args, const char *str)
-{
- int i, flag = 0, n_tags;
- char **tags = hts_readlist(str, 0, &n_tags);
- for(i=0; i<n_tags; i++)
- {
- if ( !strcasecmp(tags[i],"AN") ) flag |= SET_AN;
- else if ( !strcasecmp(tags[i],"AC") ) flag |= SET_AC;
- else if ( !strcasecmp(tags[i],"NS") ) flag |= SET_NS;
- else if ( !strcasecmp(tags[i],"AC_Hom") ) flag |= SET_AC_Hom;
- else if ( !strcasecmp(tags[i],"AC_Het") ) flag |= SET_AC_Het;
- else if ( !strcasecmp(tags[i],"AC_Hemi") ) flag |= SET_AC_Hemi;
- else if ( !strcasecmp(tags[i],"AF") ) flag |= SET_AF;
- else if ( !strcasecmp(tags[i],"MAF") ) flag |= SET_MAF;
- else if ( !strcasecmp(tags[i],"HWE") ) flag |= SET_HWE;
- else if ( !strcasecmp(tags[i],"ExcHet") ) flag |= SET_ExcHet;
- else
- {
- fprintf(stderr,"Error parsing \"--tags %s\": the tag \"%s\" is not supported\n", str,tags[i]);
- exit(1);
- }
- free(tags[i]);
- }
- if (n_tags) free(tags);
- return flag;
-}
-
-void hdr_append(args_t *args, char *fmt)
-{
- int i;
- for (i=0; i<args->npop; i++)
- bcf_hdr_printf(args->out_hdr, fmt, args->pop[i].suffix,*args->pop[i].name ? " in " : "",args->pop[i].name);
-}
-
-void list_tags(void)
-{
- error(
- "INFO/AN Number:1 Type:Integer .. Total number of alleles in called genotypes\n"
- "INFO/AC Number:A Type:Integer .. Allele count in genotypes\n"
- "INFO/NS Number:1 Type:Integer .. Number of samples with data\n"
- "INFO/AC_Hom Number:A Type:Integer .. Allele counts in homozygous genotypes\n"
- "INFO/AC_Het Number:A Type:Integer .. Allele counts in heterozygous genotypes\n"
- "INFO/AC_Hemi Number:A Type:Integer .. Allele counts in hemizygous genotypes\n"
- "INFO/AF Number:A Type:Float .. Allele frequency\n"
- "INFO/MAF Number:A Type:Float .. Minor Allele frequency\n"
- "INFO/HWE Number:A Type:Float .. HWE test (PMID:15789306)\n"
- "INFO/ExcHet Number:A Type:Float .. Probability of excess heterozygosity\n"
- );
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- args = (args_t*) calloc(1,sizeof(args_t));
- args->in_hdr = in;
- args->out_hdr = out;
- char *samples_fname = NULL;
- static struct option loptions[] =
- {
- {"list-tags",0,0,'l'},
- {"drop-missing",0,0,'d'},
- {"tags",1,0,'t'},
- {"samples-file",1,0,'S'},
- {0,0,0,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?ht:dS:l",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'l': list_tags(); break;
- case 'd': args->drop_missing = 1; break;
- case 't': args->tags |= parse_tags(args,optarg); break;
- case 'S': samples_fname = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
-
- if ( optind != argc ) error("%s",usage());
-
- args->gt_id = bcf_hdr_id2int(args->in_hdr,BCF_DT_ID,"GT");
- if ( args->gt_id<0 ) error("Error: GT field is not present\n");
-
- if ( !args->tags )
- for (c=0; c<=9; c++) args->tags |= 1<<c; // by default all tags will be filled
-
- if ( samples_fname ) parse_samples(args, samples_fname);
- init_pops(args);
-
- if ( args->tags & SET_AN ) hdr_append(args, "##INFO=<ID=AN%s,Number=1,Type=Integer,Description=\"Total number of alleles in called genotypes%s%s\">");
- if ( args->tags & SET_AC ) hdr_append(args, "##INFO=<ID=AC%s,Number=A,Type=Integer,Description=\"Allele count in genotypes%s%s\">");
- if ( args->tags & SET_NS ) hdr_append(args, "##INFO=<ID=NS%s,Number=1,Type=Integer,Description=\"Number of samples with data%s%s\">");
- if ( args->tags & SET_AC_Hom ) hdr_append(args, "##INFO=<ID=AC_Hom%s,Number=A,Type=Integer,Description=\"Allele counts in homozygous genotypes%s%s\">");
- if ( args->tags & SET_AC_Het ) hdr_append(args, "##INFO=<ID=AC_Het%s,Number=A,Type=Integer,Description=\"Allele counts in heterozygous genotypes%s%s\">");
- if ( args->tags & SET_AC_Hemi ) hdr_append(args, "##INFO=<ID=AC_Hemi%s,Number=A,Type=Integer,Description=\"Allele counts in hemizygous genotypes%s%s\">");
- if ( args->tags & SET_AF ) hdr_append(args, "##INFO=<ID=AF%s,Number=A,Type=Float,Description=\"Allele frequency%s%s\">");
- if ( args->tags & SET_MAF ) hdr_append(args, "##INFO=<ID=MAF%s,Number=A,Type=Float,Description=\"Minor Allele frequency%s%s\">");
- if ( args->tags & SET_HWE ) hdr_append(args, "##INFO=<ID=HWE%s,Number=A,Type=Float,Description=\"HWE test%s%s (PMID:15789306)\">");
- if ( args->tags & SET_ExcHet ) hdr_append(args, "##INFO=<ID=ExcHet%s,Number=A,Type=Float,Description=\"Probability of excess heterozygosity\">");
-
- return 0;
-}
-
-/*
- Wigginton 2005, PMID: 15789306
-
- nref .. number of reference alleles
- nalt .. number of alt alleles
- nhet .. number of het genotypes, assuming number of genotypes = (nref+nalt)*2
-
-*/
-
-void calc_hwe(args_t *args, int nref, int nalt, int nhet, float *p_hwe, float *p_exc_het)
-{
- int ngt = (nref+nalt) / 2;
- int nrare = nref < nalt ? nref : nalt;
-
- // sanity check: there is odd/even number of rare alleles iff there is odd/even number of hets
- if ( (nrare & 1) ^ (nhet & 1) ) error("nrare/nhet should be both odd or even: nrare=%d nref=%d nalt=%d nhet=%d\n",nrare,nref,nalt,nhet);
- if ( nrare < nhet ) error("Fewer rare alleles than hets? nrare=%d nref=%d nalt=%d nhet=%d\n",nrare,nref,nalt,nhet);
- if ( (nref+nalt) & 1 ) error("Expected diploid genotypes: nref=%d nalt=%d\n",nref,nalt);
-
- // initialize het probs
- hts_expand(double,nrare+1,args->mhwe_probs,args->hwe_probs);
- memset(args->hwe_probs, 0, sizeof(*args->hwe_probs)*(nrare+1));
- double *probs = args->hwe_probs;
-
- // start at midpoint
- int mid = nrare * (nref + nalt - nrare) / (nref + nalt);
-
- // check to ensure that midpoint and rare alleles have same parity
- if ( (nrare & 1) ^ (mid & 1) ) mid++;
-
- int het = mid;
- int hom_r = (nrare - mid) / 2;
- int hom_c = ngt - het - hom_r;
- double sum = probs[mid] = 1.0;
-
- for (het = mid; het > 1; het -= 2)
- {
- probs[het - 2] = probs[het] * het * (het - 1.0) / (4.0 * (hom_r + 1.0) * (hom_c + 1.0));
- sum += probs[het - 2];
-
- // 2 fewer heterozygotes for next iteration -> add one rare, one common homozygote
- hom_r++;
- hom_c++;
- }
-
- het = mid;
- hom_r = (nrare - mid) / 2;
- hom_c = ngt - het - hom_r;
- for (het = mid; het <= nrare - 2; het += 2)
- {
- probs[het + 2] = probs[het] * 4.0 * hom_r * hom_c / ((het + 2.0) * (het + 1.0));
- sum += probs[het + 2];
-
- // add 2 heterozygotes for next iteration -> subtract one rare, one common homozygote
- hom_r--;
- hom_c--;
- }
-
- for (het=0; het<nrare+1; het++) probs[het] /= sum;
-
- double prob = probs[nhet];
- for (het = nhet + 1; het <= nrare; het++) prob += probs[het];
- *p_exc_het = prob;
-
- prob = 0;
- for (het=0; het <= nrare; het++)
- {
- if ( probs[het] > probs[nhet]) continue;
- prob += probs[het];
- }
- if ( prob > 1 ) prob = 1;
- *p_hwe = prob;
-}
-
-static inline void set_counts(pop_t *pop, int is_half, int is_hom, int is_hemi, int als)
-{
- int ial;
- for (ial=0; als; ial++)
- {
- if ( als&1 )
- {
- if ( is_half ) pop->counts[ial].nac++;
- else if ( !is_hom ) pop->counts[ial].nhet++;
- else if ( !is_hemi ) pop->counts[ial].nhom += 2;
- else pop->counts[ial].nhemi++;
- }
- als >>= 1;
- }
- pop->ns++;
-}
-static void clean_counts(pop_t *pop, int nals)
-{
- pop->ns = 0;
- memset(pop->counts,0,sizeof(counts_t)*nals);
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int i,j, nsmpl = bcf_hdr_nsamples(args->in_hdr);;
-
- bcf_unpack(rec, BCF_UN_FMT);
- bcf_fmt_t *fmt_gt = NULL;
- for (i=0; i<rec->n_fmt; i++)
- if ( rec->d.fmt[i].id==args->gt_id ) { fmt_gt = &rec->d.fmt[i]; break; }
- if ( !fmt_gt ) return rec; // no GT tag
-
- hts_expand(int32_t,rec->n_allele, args->miarr, args->iarr);
- hts_expand(float,rec->n_allele*2, args->mfarr, args->farr);
- for (i=0; i<args->npop; i++)
- hts_expand(counts_t,rec->n_allele,args->pop[i].mcounts, args->pop[i].counts);
-
- for (i=0; i<args->npop; i++)
- clean_counts(&args->pop[i], rec->n_allele);
-
- assert( rec->n_allele < 8*sizeof(int) );
-
- #define BRANCH_INT(type_t,vector_end) \
- { \
- for (i=0; i<nsmpl; i++) \
- { \
- type_t *p = (type_t*) (fmt_gt->p + i*fmt_gt->size); \
- int ial, als = 0, nals = 0, is_half, is_hom, is_hemi; \
- for (ial=0; ial<fmt_gt->n; ial++) \
- { \
- if ( p[ial]==vector_end ) break; /* smaller ploidy */ \
- if ( bcf_gt_is_missing(p[ial]) ) continue; /* missing allele */ \
- int idx = bcf_gt_allele(p[ial]); \
- nals++; \
- \
- if ( idx >= rec->n_allele ) \
- error("Incorrect allele (\"%d\") in %s at %s:%d\n",idx,args->in_hdr->samples[i],bcf_seqname(args->in_hdr,rec),rec->pos+1); \
- als |= (1<<idx); /* this breaks with too many alleles */ \
- } \
- if ( nals==0 ) continue; /* missing genotype */ \
- is_hom = als && !(als & (als-1)); /* only one bit is set */ \
- if ( nals!=ial ) \
- { \
- if ( args->drop_missing ) is_hemi = 0, is_half = 1; \
- else is_hemi = 1, is_half = 0; \
- } \
- else if ( nals==1 ) is_hemi = 1, is_half = 0; \
- else is_hemi = 0, is_half = 0; \
- pop_t **pop = &args->smpl2pop[i*(args->npop+1)]; \
- while ( *pop ) { set_counts(*pop,is_half,is_hom,is_hemi,als); pop++; }\
- } \
- }
- switch (fmt_gt->type) {
- case BCF_BT_INT8: BRANCH_INT(int8_t, bcf_int8_vector_end); break;
- case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_vector_end); break;
- case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_vector_end); break;
- default: error("The GT type is not recognised: %d at %s:%d\n",fmt_gt->type, bcf_seqname(args->in_hdr,rec),rec->pos+1); break;
- }
- #undef BRANCH_INT
-
- if ( args->tags & SET_NS )
- {
- for (i=0; i<args->npop; i++)
- {
- args->str.l = 0;
- ksprintf(&args->str, "NS%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,&args->pop[i].ns,1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & SET_AN )
- {
- for (i=0; i<args->npop; i++)
- {
- pop_t *pop = &args->pop[i];
- int32_t an = 0;
- for (j=0; j<rec->n_allele; j++)
- an += pop->counts[j].nhet + pop->counts[j].nhom + pop->counts[j].nhemi + pop->counts[j].nac;
-
- args->str.l = 0;
- ksprintf(&args->str, "AN%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,&an,1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & (SET_AF | SET_MAF) )
- {
- for (i=0; i<args->npop; i++)
- {
- int32_t an = 0;
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->farr, 0, sizeof(*args->farr)*(rec->n_allele-1));
- for (j=1; j<rec->n_allele; j++)
- args->farr[j-1] += pop->counts[j].nhet + pop->counts[j].nhom + pop->counts[j].nhemi + pop->counts[j].nac;
- an = pop->counts[0].nhet + pop->counts[0].nhom + pop->counts[0].nhemi + pop->counts[0].nac;
- for (j=1; j<rec->n_allele; j++) an += args->farr[j-1];
- if ( !an ) continue;
- for (j=1; j<rec->n_allele; j++) args->farr[j-1] /= an;
- }
- if ( args->tags & SET_AF )
- {
- args->str.l = 0;
- ksprintf(&args->str, "AF%s", args->pop[i].suffix);
- if ( bcf_update_info_float(args->out_hdr,rec,args->str.s,args->farr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- if ( args->tags & SET_MAF )
- {
- if ( !an ) continue;
- for (j=1; j<rec->n_allele; j++)
- if ( args->farr[j-1] > 0.5 ) args->farr[j-1] = 1 - args->farr[j-1]; // todo: this is incorrect for multiallelic sites
- args->str.l = 0;
- ksprintf(&args->str, "MAF%s", args->pop[i].suffix);
- if ( bcf_update_info_float(args->out_hdr,rec,args->str.s,args->farr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- }
- if ( args->tags & SET_AC )
- {
- for (i=0; i<args->npop; i++)
- {
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->iarr, 0, sizeof(*args->iarr)*(rec->n_allele-1));
- for (j=1; j<rec->n_allele; j++)
- args->iarr[j-1] += pop->counts[j].nhet + pop->counts[j].nhom + pop->counts[j].nhemi + pop->counts[j].nac;
- }
- args->str.l = 0;
- ksprintf(&args->str, "AC%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,args->iarr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & SET_AC_Het )
- {
- for (i=0; i<args->npop; i++)
- {
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->iarr, 0, sizeof(*args->iarr)*(rec->n_allele-1));
- for (j=1; j<rec->n_allele; j++)
- args->iarr[j-1] += pop->counts[j].nhet;
- }
- args->str.l = 0;
- ksprintf(&args->str, "AC_Het%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,args->iarr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & SET_AC_Hom )
- {
- for (i=0; i<args->npop; i++)
- {
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->iarr, 0, sizeof(*args->iarr)*(rec->n_allele-1));
- for (j=1; j<rec->n_allele; j++)
- args->iarr[j-1] += pop->counts[j].nhom;
- }
- args->str.l = 0;
- ksprintf(&args->str, "AC_Hom%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,args->iarr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & SET_AC_Hemi && rec->n_allele > 1 )
- {
- for (i=0; i<args->npop; i++)
- {
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->iarr, 0, sizeof(*args->iarr)*(rec->n_allele-1));
- for (j=1; j<rec->n_allele; j++)
- args->iarr[j-1] += pop->counts[j].nhemi;
- }
- args->str.l = 0;
- ksprintf(&args->str, "AC_Hemi%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,args->iarr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & (SET_HWE|SET_ExcHet) )
- {
- for (i=0; i<args->npop; i++)
- {
- float *fhwe = args->farr;
- float *fexc_het = args->farr + rec->n_allele;
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->farr, 0, sizeof(*args->farr)*(2*rec->n_allele));
- int nref_tot = pop->counts[0].nhom;
- for (j=0; j<rec->n_allele; j++) nref_tot += pop->counts[j].nhet; // NB this neglects multiallelic genotypes
- for (j=1; j<rec->n_allele; j++)
- {
- int nref = nref_tot - pop->counts[j].nhet;
- int nalt = pop->counts[j].nhet + pop->counts[j].nhom;
- int nhet = pop->counts[j].nhet;
- if ( nref>0 && nalt>0 )
- calc_hwe(args, nref, nalt, nhet, &fhwe[j-1], &fexc_het[j-1]);
- else
- fhwe[j-1] = fexc_het[j-1] = 1;
- }
- }
- if ( args->tags & SET_HWE )
- {
- args->str.l = 0;
- ksprintf(&args->str, "HWE%s", args->pop[i].suffix);
- if ( bcf_update_info_float(args->out_hdr,rec,args->str.s,fhwe,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- if ( args->tags & SET_ExcHet )
- {
- args->str.l = 0;
- ksprintf(&args->str, "ExcHet%s", args->pop[i].suffix);
- if ( bcf_update_info_float(args->out_hdr,rec,args->str.s,fexc_het,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- }
-
- return rec;
-}
-
-void destroy(void)
-{
- int i;
- for (i=0; i<args->npop; i++)
- {
- free(args->pop[i].name);
- free(args->pop[i].suffix);
- free(args->pop[i].smpl);
- free(args->pop[i].counts);
- }
- free(args->str.s);
- free(args->pop);
- free(args->smpl2pop);
- free(args->iarr);
- free(args->farr);
- free(args->hwe_probs);
- free(args);
-}
-
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2015 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/kseq.h>
-#include <htslib/vcf.h>
-#include <htslib/khash_str2int.h>
-#include "bcftools.h"
-
-#define SET_AN (1<<0)
-#define SET_AC (1<<1)
-#define SET_AC_Hom (1<<2)
-#define SET_AC_Het (1<<3)
-#define SET_AC_Hemi (1<<4)
-#define SET_AF (1<<5)
-#define SET_NS (1<<6)
-#define SET_MAF (1<<7)
-#define SET_HWE (1<<8)
-#define SET_ExcHet (1<<9)
-
-typedef struct
-{
- int nhom, nhet, nhemi, nac;
-}
-counts_t;
-
-typedef struct
-{
- int ns;
- int ncounts, mcounts;
- counts_t *counts;
- char *name, *suffix;
- int nsmpl, *smpl;
-}
-pop_t;
-
-typedef struct
-{
- bcf_hdr_t *in_hdr, *out_hdr;
- int npop, tags, drop_missing, gt_id;
- pop_t *pop, **smpl2pop;
- float *farr;
- int32_t *iarr, niarr, miarr, nfarr, mfarr;
- double *hwe_probs;
- int mhwe_probs;
- kstring_t str;
-}
-args_t;
-
-static args_t *args;
-
-const char *about(void)
-{
- return "Set INFO tags AF, AC, AC_Hemi, AC_Hom, AC_Het, AN, ExcHet, HWE, MAF, NS.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Set INFO tags AF, AC, AC_Hemi, AC_Hom, AC_Het, AN, ExcHet, HWE, MAF, NS.\n"
- "Usage: bcftools +fill-tags [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -d, --drop-missing do not count half-missing genotypes \"./1\" as hemizygous\n"
- " -l, --list-tags list available tags with description\n"
- " -t, --tags LIST list of output tags. By default, all tags are filled.\n"
- " -S, --samples-file FILE list of samples (first column) and comma-separated list of populations (second column)\n"
- "\n"
- "Example:\n"
- " bcftools +fill-tags in.bcf -Ob -o out.bcf\n"
- " bcftools +fill-tags in.bcf -Ob -o out.bcf -- -t AN,AC\n"
- " bcftools +fill-tags in.bcf -Ob -o out.bcf -- -d\n"
- " bcftools +fill-tags in.bcf -Ob -o out.bcf -- -S sample-group.txt -t HWE\n"
- "\n";
-}
-
-void parse_samples(args_t *args, char *fname)
-{
- htsFile *fp = hts_open(fname, "r");
- if ( !fp ) error("Could not read: %s\n", fname);
-
- void *pop2i = khash_str2int_init();
- void *smpli = khash_str2int_init();
- kstring_t str = {0,0,0};
-
- int moff = 0, *off = NULL, nsmpl = 0;
- while ( hts_getline(fp, KS_SEP_LINE, &str)>=0 )
- {
- // NA12400 GRP1
- // NA18507 GRP1,GRP2
- char *pop_names = str.s + str.l - 1;
- while ( pop_names >= str.s && isspace(*pop_names) ) pop_names--;
- if ( pop_names <= str.s ) error("Could not parse the file: %s\n", str.s);
- pop_names[1] = 0; // trailing spaces
- while ( pop_names >= str.s && !isspace(*pop_names) ) pop_names--;
- if ( pop_names <= str.s ) error("Could not parse the file: %s\n", str.s);
-
- char *smpl = pop_names++;
- while ( smpl >= str.s && isspace(*smpl) ) smpl--;
- if ( smpl <= str.s+1 ) error("Could not parse the file: %s\n", str.s);
- smpl[1] = 0;
- smpl = str.s;
-
- int ismpl = bcf_hdr_id2int(args->in_hdr,BCF_DT_SAMPLE,smpl);
- if ( ismpl<0 )
- {
- fprintf(bcftools_stderr,"Warning: The sample not present in the VCF: %s\n",smpl);
- continue;
- }
- if ( khash_str2int_has_key(smpli,smpl) )
- {
- fprintf(bcftools_stderr,"Warning: The sample is listed twice in %s: %s\n",fname,smpl);
- continue;
- }
- khash_str2int_inc(smpli,strdup(smpl));
-
- int i,npops = ksplit_core(pop_names,',',&moff,&off);
- for (i=0; i<npops; i++)
- {
- char *pop_name = &pop_names[off[i]];
- if ( !khash_str2int_has_key(pop2i,pop_name) )
- {
- pop_name = strdup(pop_name);
- khash_str2int_set(pop2i,pop_name,args->npop);
- args->npop++;
- args->pop = (pop_t*) realloc(args->pop,args->npop*sizeof(*args->pop));
- memset(args->pop+args->npop-1,0,sizeof(*args->pop));
- args->pop[args->npop-1].name = pop_name;
- args->pop[args->npop-1].suffix = (char*)malloc(strlen(pop_name)+2);
- memcpy(args->pop[args->npop-1].suffix+1,pop_name,strlen(pop_name)+1);
- args->pop[args->npop-1].suffix[0] = '_';
- }
- int ipop = 0;
- khash_str2int_get(pop2i,pop_name,&ipop);
- pop_t *pop = &args->pop[ipop];
- pop->nsmpl++;
- pop->smpl = (int*) realloc(pop->smpl,pop->nsmpl*sizeof(*pop->smpl));
- pop->smpl[pop->nsmpl-1] = ismpl;
- }
- nsmpl++;
- }
-
- if ( nsmpl != bcf_hdr_nsamples(args->in_hdr) )
- fprintf(bcftools_stderr,"Warning: %d samples in the list, %d samples in the VCF.\n", nsmpl,bcf_hdr_nsamples(args->in_hdr));
-
- if ( !args->npop ) error("No populations given?\n");
-
- khash_str2int_destroy(pop2i);
- khash_str2int_destroy_free(smpli);
- free(str.s);
- free(off);
- hts_close(fp);
-}
-
-void init_pops(args_t *args)
-{
- int i,j, nsmpl;
-
- // add the population "ALL", which is a summary population for all samples
- args->npop++;
- args->pop = (pop_t*) realloc(args->pop,args->npop*sizeof(*args->pop));
- memset(args->pop+args->npop-1,0,sizeof(*args->pop));
- args->pop[args->npop-1].name = strdup("");
- args->pop[args->npop-1].suffix = strdup("");
-
- nsmpl = bcf_hdr_nsamples(args->in_hdr);
- args->smpl2pop = (pop_t**) calloc(nsmpl*(args->npop+1),sizeof(pop_t*));
- for (i=0; i<nsmpl; i++)
- args->smpl2pop[i*(args->npop+1)] = &args->pop[args->npop-1];
-
- for (i=0; i<args->npop; i++)
- {
- for (j=0; j<args->pop[i].nsmpl; j++)
- {
- int ismpl = args->pop[i].smpl[j];
- pop_t **smpl2pop = &args->smpl2pop[ismpl*(args->npop+1)];
- while (*smpl2pop) smpl2pop++;
- *smpl2pop = &args->pop[i];
- }
- }
-}
-
-int parse_tags(args_t *args, const char *str)
-{
- int i, flag = 0, n_tags;
- char **tags = hts_readlist(str, 0, &n_tags);
- for(i=0; i<n_tags; i++)
- {
- if ( !strcasecmp(tags[i],"AN") ) flag |= SET_AN;
- else if ( !strcasecmp(tags[i],"AC") ) flag |= SET_AC;
- else if ( !strcasecmp(tags[i],"NS") ) flag |= SET_NS;
- else if ( !strcasecmp(tags[i],"AC_Hom") ) flag |= SET_AC_Hom;
- else if ( !strcasecmp(tags[i],"AC_Het") ) flag |= SET_AC_Het;
- else if ( !strcasecmp(tags[i],"AC_Hemi") ) flag |= SET_AC_Hemi;
- else if ( !strcasecmp(tags[i],"AF") ) flag |= SET_AF;
- else if ( !strcasecmp(tags[i],"MAF") ) flag |= SET_MAF;
- else if ( !strcasecmp(tags[i],"HWE") ) flag |= SET_HWE;
- else if ( !strcasecmp(tags[i],"ExcHet") ) flag |= SET_ExcHet;
- else
- {
- fprintf(bcftools_stderr,"Error parsing \"--tags %s\": the tag \"%s\" is not supported\n", str,tags[i]);
- exit(1);
- }
- free(tags[i]);
- }
- if (n_tags) free(tags);
- return flag;
-}
-
-void hdr_append(args_t *args, char *fmt)
-{
- int i;
- for (i=0; i<args->npop; i++)
- bcf_hdr_printf(args->out_hdr, fmt, args->pop[i].suffix,*args->pop[i].name ? " in " : "",args->pop[i].name);
-}
-
-void list_tags(void)
-{
- error(
- "INFO/AN Number:1 Type:Integer .. Total number of alleles in called genotypes\n"
- "INFO/AC Number:A Type:Integer .. Allele count in genotypes\n"
- "INFO/NS Number:1 Type:Integer .. Number of samples with data\n"
- "INFO/AC_Hom Number:A Type:Integer .. Allele counts in homozygous genotypes\n"
- "INFO/AC_Het Number:A Type:Integer .. Allele counts in heterozygous genotypes\n"
- "INFO/AC_Hemi Number:A Type:Integer .. Allele counts in hemizygous genotypes\n"
- "INFO/AF Number:A Type:Float .. Allele frequency\n"
- "INFO/MAF Number:A Type:Float .. Minor Allele frequency\n"
- "INFO/HWE Number:A Type:Float .. HWE test (PMID:15789306)\n"
- "INFO/ExcHet Number:A Type:Float .. Probability of excess heterozygosity\n"
- );
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- args = (args_t*) calloc(1,sizeof(args_t));
- args->in_hdr = in;
- args->out_hdr = out;
- char *samples_fname = NULL;
- static struct option loptions[] =
- {
- {"list-tags",0,0,'l'},
- {"drop-missing",0,0,'d'},
- {"tags",1,0,'t'},
- {"samples-file",1,0,'S'},
- {0,0,0,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?ht:dS:l",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'l': list_tags(); break;
- case 'd': args->drop_missing = 1; break;
- case 't': args->tags |= parse_tags(args,optarg); break;
- case 'S': samples_fname = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
-
- if ( optind != argc ) error("%s",usage());
-
- args->gt_id = bcf_hdr_id2int(args->in_hdr,BCF_DT_ID,"GT");
- if ( args->gt_id<0 ) error("Error: GT field is not present\n");
-
- if ( !args->tags )
- for (c=0; c<=9; c++) args->tags |= 1<<c; // by default all tags will be filled
-
- if ( samples_fname ) parse_samples(args, samples_fname);
- init_pops(args);
-
- if ( args->tags & SET_AN ) hdr_append(args, "##INFO=<ID=AN%s,Number=1,Type=Integer,Description=\"Total number of alleles in called genotypes%s%s\">");
- if ( args->tags & SET_AC ) hdr_append(args, "##INFO=<ID=AC%s,Number=A,Type=Integer,Description=\"Allele count in genotypes%s%s\">");
- if ( args->tags & SET_NS ) hdr_append(args, "##INFO=<ID=NS%s,Number=1,Type=Integer,Description=\"Number of samples with data%s%s\">");
- if ( args->tags & SET_AC_Hom ) hdr_append(args, "##INFO=<ID=AC_Hom%s,Number=A,Type=Integer,Description=\"Allele counts in homozygous genotypes%s%s\">");
- if ( args->tags & SET_AC_Het ) hdr_append(args, "##INFO=<ID=AC_Het%s,Number=A,Type=Integer,Description=\"Allele counts in heterozygous genotypes%s%s\">");
- if ( args->tags & SET_AC_Hemi ) hdr_append(args, "##INFO=<ID=AC_Hemi%s,Number=A,Type=Integer,Description=\"Allele counts in hemizygous genotypes%s%s\">");
- if ( args->tags & SET_AF ) hdr_append(args, "##INFO=<ID=AF%s,Number=A,Type=Float,Description=\"Allele frequency%s%s\">");
- if ( args->tags & SET_MAF ) hdr_append(args, "##INFO=<ID=MAF%s,Number=A,Type=Float,Description=\"Minor Allele frequency%s%s\">");
- if ( args->tags & SET_HWE ) hdr_append(args, "##INFO=<ID=HWE%s,Number=A,Type=Float,Description=\"HWE test%s%s (PMID:15789306)\">");
- if ( args->tags & SET_ExcHet ) hdr_append(args, "##INFO=<ID=ExcHet%s,Number=A,Type=Float,Description=\"Probability of excess heterozygosity\">");
-
- return 0;
-}
-
-/*
- Wigginton 2005, PMID: 15789306
-
- nref .. number of reference alleles
- nalt .. number of alt alleles
- nhet .. number of het genotypes, assuming number of genotypes = (nref+nalt)*2
-
-*/
-
-void calc_hwe(args_t *args, int nref, int nalt, int nhet, float *p_hwe, float *p_exc_het)
-{
- int ngt = (nref+nalt) / 2;
- int nrare = nref < nalt ? nref : nalt;
-
- // sanity check: there is odd/even number of rare alleles iff there is odd/even number of hets
- if ( (nrare & 1) ^ (nhet & 1) ) error("nrare/nhet should be both odd or even: nrare=%d nref=%d nalt=%d nhet=%d\n",nrare,nref,nalt,nhet);
- if ( nrare < nhet ) error("Fewer rare alleles than hets? nrare=%d nref=%d nalt=%d nhet=%d\n",nrare,nref,nalt,nhet);
- if ( (nref+nalt) & 1 ) error("Expected diploid genotypes: nref=%d nalt=%d\n",nref,nalt);
-
- // initialize het probs
- hts_expand(double,nrare+1,args->mhwe_probs,args->hwe_probs);
- memset(args->hwe_probs, 0, sizeof(*args->hwe_probs)*(nrare+1));
- double *probs = args->hwe_probs;
-
- // start at midpoint
- int mid = nrare * (nref + nalt - nrare) / (nref + nalt);
-
- // check to ensure that midpoint and rare alleles have same parity
- if ( (nrare & 1) ^ (mid & 1) ) mid++;
-
- int het = mid;
- int hom_r = (nrare - mid) / 2;
- int hom_c = ngt - het - hom_r;
- double sum = probs[mid] = 1.0;
-
- for (het = mid; het > 1; het -= 2)
- {
- probs[het - 2] = probs[het] * het * (het - 1.0) / (4.0 * (hom_r + 1.0) * (hom_c + 1.0));
- sum += probs[het - 2];
-
- // 2 fewer heterozygotes for next iteration -> add one rare, one common homozygote
- hom_r++;
- hom_c++;
- }
-
- het = mid;
- hom_r = (nrare - mid) / 2;
- hom_c = ngt - het - hom_r;
- for (het = mid; het <= nrare - 2; het += 2)
- {
- probs[het + 2] = probs[het] * 4.0 * hom_r * hom_c / ((het + 2.0) * (het + 1.0));
- sum += probs[het + 2];
-
- // add 2 heterozygotes for next iteration -> subtract one rare, one common homozygote
- hom_r--;
- hom_c--;
- }
-
- for (het=0; het<nrare+1; het++) probs[het] /= sum;
-
- double prob = probs[nhet];
- for (het = nhet + 1; het <= nrare; het++) prob += probs[het];
- *p_exc_het = prob;
-
- prob = 0;
- for (het=0; het <= nrare; het++)
- {
- if ( probs[het] > probs[nhet]) continue;
- prob += probs[het];
- }
- if ( prob > 1 ) prob = 1;
- *p_hwe = prob;
-}
-
-static inline void set_counts(pop_t *pop, int is_half, int is_hom, int is_hemi, int als)
-{
- int ial;
- for (ial=0; als; ial++)
- {
- if ( als&1 )
- {
- if ( is_half ) pop->counts[ial].nac++;
- else if ( !is_hom ) pop->counts[ial].nhet++;
- else if ( !is_hemi ) pop->counts[ial].nhom += 2;
- else pop->counts[ial].nhemi++;
- }
- als >>= 1;
- }
- pop->ns++;
-}
-static void clean_counts(pop_t *pop, int nals)
-{
- pop->ns = 0;
- memset(pop->counts,0,sizeof(counts_t)*nals);
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int i,j, nsmpl = bcf_hdr_nsamples(args->in_hdr);;
-
- bcf_unpack(rec, BCF_UN_FMT);
- bcf_fmt_t *fmt_gt = NULL;
- for (i=0; i<rec->n_fmt; i++)
- if ( rec->d.fmt[i].id==args->gt_id ) { fmt_gt = &rec->d.fmt[i]; break; }
- if ( !fmt_gt ) return rec; // no GT tag
-
- hts_expand(int32_t,rec->n_allele, args->miarr, args->iarr);
- hts_expand(float,rec->n_allele*2, args->mfarr, args->farr);
- for (i=0; i<args->npop; i++)
- hts_expand(counts_t,rec->n_allele,args->pop[i].mcounts, args->pop[i].counts);
-
- for (i=0; i<args->npop; i++)
- clean_counts(&args->pop[i], rec->n_allele);
-
- assert( rec->n_allele < 8*sizeof(int) );
-
- #define BRANCH_INT(type_t,vector_end) \
- { \
- for (i=0; i<nsmpl; i++) \
- { \
- type_t *p = (type_t*) (fmt_gt->p + i*fmt_gt->size); \
- int ial, als = 0, nals = 0, is_half, is_hom, is_hemi; \
- for (ial=0; ial<fmt_gt->n; ial++) \
- { \
- if ( p[ial]==vector_end ) break; /* smaller ploidy */ \
- if ( bcf_gt_is_missing(p[ial]) ) continue; /* missing allele */ \
- int idx = bcf_gt_allele(p[ial]); \
- nals++; \
- \
- if ( idx >= rec->n_allele ) \
- error("Incorrect allele (\"%d\") in %s at %s:%d\n",idx,args->in_hdr->samples[i],bcf_seqname(args->in_hdr,rec),rec->pos+1); \
- als |= (1<<idx); /* this breaks with too many alleles */ \
- } \
- if ( nals==0 ) continue; /* missing genotype */ \
- is_hom = als && !(als & (als-1)); /* only one bit is set */ \
- if ( nals!=ial ) \
- { \
- if ( args->drop_missing ) is_hemi = 0, is_half = 1; \
- else is_hemi = 1, is_half = 0; \
- } \
- else if ( nals==1 ) is_hemi = 1, is_half = 0; \
- else is_hemi = 0, is_half = 0; \
- pop_t **pop = &args->smpl2pop[i*(args->npop+1)]; \
- while ( *pop ) { set_counts(*pop,is_half,is_hom,is_hemi,als); pop++; }\
- } \
- }
- switch (fmt_gt->type) {
- case BCF_BT_INT8: BRANCH_INT(int8_t, bcf_int8_vector_end); break;
- case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_vector_end); break;
- case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_vector_end); break;
- default: error("The GT type is not recognised: %d at %s:%d\n",fmt_gt->type, bcf_seqname(args->in_hdr,rec),rec->pos+1); break;
- }
- #undef BRANCH_INT
-
- if ( args->tags & SET_NS )
- {
- for (i=0; i<args->npop; i++)
- {
- args->str.l = 0;
- ksprintf(&args->str, "NS%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,&args->pop[i].ns,1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & SET_AN )
- {
- for (i=0; i<args->npop; i++)
- {
- pop_t *pop = &args->pop[i];
- int32_t an = 0;
- for (j=0; j<rec->n_allele; j++)
- an += pop->counts[j].nhet + pop->counts[j].nhom + pop->counts[j].nhemi + pop->counts[j].nac;
-
- args->str.l = 0;
- ksprintf(&args->str, "AN%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,&an,1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & (SET_AF | SET_MAF) )
- {
- for (i=0; i<args->npop; i++)
- {
- int32_t an = 0;
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->farr, 0, sizeof(*args->farr)*(rec->n_allele-1));
- for (j=1; j<rec->n_allele; j++)
- args->farr[j-1] += pop->counts[j].nhet + pop->counts[j].nhom + pop->counts[j].nhemi + pop->counts[j].nac;
- an = pop->counts[0].nhet + pop->counts[0].nhom + pop->counts[0].nhemi + pop->counts[0].nac;
- for (j=1; j<rec->n_allele; j++) an += args->farr[j-1];
- if ( !an ) continue;
- for (j=1; j<rec->n_allele; j++) args->farr[j-1] /= an;
- }
- if ( args->tags & SET_AF )
- {
- args->str.l = 0;
- ksprintf(&args->str, "AF%s", args->pop[i].suffix);
- if ( bcf_update_info_float(args->out_hdr,rec,args->str.s,args->farr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- if ( args->tags & SET_MAF )
- {
- if ( !an ) continue;
- for (j=1; j<rec->n_allele; j++)
- if ( args->farr[j-1] > 0.5 ) args->farr[j-1] = 1 - args->farr[j-1]; // todo: this is incorrect for multiallelic sites
- args->str.l = 0;
- ksprintf(&args->str, "MAF%s", args->pop[i].suffix);
- if ( bcf_update_info_float(args->out_hdr,rec,args->str.s,args->farr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- }
- if ( args->tags & SET_AC )
- {
- for (i=0; i<args->npop; i++)
- {
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->iarr, 0, sizeof(*args->iarr)*(rec->n_allele-1));
- for (j=1; j<rec->n_allele; j++)
- args->iarr[j-1] += pop->counts[j].nhet + pop->counts[j].nhom + pop->counts[j].nhemi + pop->counts[j].nac;
- }
- args->str.l = 0;
- ksprintf(&args->str, "AC%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,args->iarr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & SET_AC_Het )
- {
- for (i=0; i<args->npop; i++)
- {
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->iarr, 0, sizeof(*args->iarr)*(rec->n_allele-1));
- for (j=1; j<rec->n_allele; j++)
- args->iarr[j-1] += pop->counts[j].nhet;
- }
- args->str.l = 0;
- ksprintf(&args->str, "AC_Het%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,args->iarr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & SET_AC_Hom )
- {
- for (i=0; i<args->npop; i++)
- {
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->iarr, 0, sizeof(*args->iarr)*(rec->n_allele-1));
- for (j=1; j<rec->n_allele; j++)
- args->iarr[j-1] += pop->counts[j].nhom;
- }
- args->str.l = 0;
- ksprintf(&args->str, "AC_Hom%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,args->iarr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & SET_AC_Hemi && rec->n_allele > 1 )
- {
- for (i=0; i<args->npop; i++)
- {
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->iarr, 0, sizeof(*args->iarr)*(rec->n_allele-1));
- for (j=1; j<rec->n_allele; j++)
- args->iarr[j-1] += pop->counts[j].nhemi;
- }
- args->str.l = 0;
- ksprintf(&args->str, "AC_Hemi%s", args->pop[i].suffix);
- if ( bcf_update_info_int32(args->out_hdr,rec,args->str.s,args->iarr,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- if ( args->tags & (SET_HWE|SET_ExcHet) )
- {
- for (i=0; i<args->npop; i++)
- {
- float *fhwe = args->farr;
- float *fexc_het = args->farr + rec->n_allele;
- if ( rec->n_allele > 1 )
- {
- pop_t *pop = &args->pop[i];
- memset(args->farr, 0, sizeof(*args->farr)*(2*rec->n_allele));
- int nref_tot = pop->counts[0].nhom;
- for (j=0; j<rec->n_allele; j++) nref_tot += pop->counts[j].nhet; // NB this neglects multiallelic genotypes
- for (j=1; j<rec->n_allele; j++)
- {
- int nref = nref_tot - pop->counts[j].nhet;
- int nalt = pop->counts[j].nhet + pop->counts[j].nhom;
- int nhet = pop->counts[j].nhet;
- if ( nref>0 && nalt>0 )
- calc_hwe(args, nref, nalt, nhet, &fhwe[j-1], &fexc_het[j-1]);
- else
- fhwe[j-1] = fexc_het[j-1] = 1;
- }
- }
- if ( args->tags & SET_HWE )
- {
- args->str.l = 0;
- ksprintf(&args->str, "HWE%s", args->pop[i].suffix);
- if ( bcf_update_info_float(args->out_hdr,rec,args->str.s,fhwe,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- if ( args->tags & SET_ExcHet )
- {
- args->str.l = 0;
- ksprintf(&args->str, "ExcHet%s", args->pop[i].suffix);
- if ( bcf_update_info_float(args->out_hdr,rec,args->str.s,fexc_het,rec->n_allele-1)!=0 )
- error("Error occurred while updating %s at %s:%d\n", args->str.s,bcf_seqname(args->in_hdr,rec),rec->pos+1);
- }
- }
- }
-
- return rec;
-}
-
-void destroy(void)
-{
- int i;
- for (i=0; i<args->npop; i++)
- {
- free(args->pop[i].name);
- free(args->pop[i].suffix);
- free(args->pop[i].smpl);
- free(args->pop[i].counts);
- }
- free(args->str.s);
- free(args->pop);
- free(args->smpl2pop);
- free(args->iarr);
- free(args->farr);
- free(args->hwe_probs);
- free(args);
-}
-
-
-
+++ /dev/null
-/*
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <ctype.h>
-#include <htslib/vcf.h>
-#include <htslib/kseq.h>
-#include "bcftools.h"
-#include "ploidy.h"
-
-static bcf_hdr_t *in_hdr = NULL, *out_hdr = NULL;
-static int *sample2sex = NULL;
-static int n_sample = 0, nsex = 0, *sex2ploidy = NULL;
-static int32_t ngt_arr = 0, *gt_arr = NULL, *gt_arr2 = NULL, ngt_arr2 = 0;
-static ploidy_t *ploidy = NULL;
-static int force_ploidy = -1;
-
-const char *about(void)
-{
- return "Fix ploidy.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Fix ploidy\n"
- "Usage: bcftools +fixploidy [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -d, --default-ploidy <int> default ploidy for regions unlisted in -p [2]\n"
- " -f, --force-ploidy <int> ignore -p, set the same ploidy for all genotypes\n"
- " -p, --ploidy <file> space/tab-delimited list of CHROM,FROM,TO,SEX,PLOIDY\n"
- " -s, --sex <file> list of samples, \"NAME SEX\"\n"
- " -t, --tags <list> VCF tags to fix [GT]\n"
- "\n"
- "Example:\n"
- " # Default ploidy, if -p not given. Unlisted regions have ploidy 2\n"
- " X 1 60000 M 1\n"
- " X 2699521 154931043 M 1\n"
- " Y 1 59373566 M 1\n"
- " Y 1 59373566 F 0\n"
- " MT 1 16569 M 1\n"
- " MT 1 16569 F 1\n"
- " \n"
- " # Example of -s file, sex of unlisted samples is \"F\"\n"
- " sampleName1 M\n"
- " \n"
- " bcftools +fixploidy in.vcf -- -s samples.txt\n"
- "\n";
-}
-
-void set_samples(char *fname, bcf_hdr_t *hdr, ploidy_t *ploidy, int *sample2sex)
-{
- kstring_t tmp = {0,0,0};
-
- htsFile *fp = hts_open(fname, "r");
- if ( !fp ) error("Could not read: %s\n", fname);
- while ( hts_getline(fp, KS_SEP_LINE, &tmp) > 0 )
- {
- char *ss = tmp.s;
- while ( *ss && isspace(*ss) ) ss++;
- if ( !*ss ) error("Could not parse: %s\n", tmp.s);
- if ( *ss=='#' ) continue;
- char *se = ss;
- while ( *se && !isspace(*se) ) se++;
- char x = *se; *se = 0;
-
- int ismpl = bcf_hdr_id2int(hdr, BCF_DT_SAMPLE, ss);
- if ( ismpl < 0 ) { fprintf(stderr,"Warning: No such sample in the VCF: %s\n",ss); continue; }
-
- *se = x;
- ss = se+1;
- while ( *ss && isspace(*ss) ) ss++;
- if ( !*ss ) error("Could not parse: %s\n", tmp.s);
- se = ss;
- while ( *se && !isspace(*se) ) se++;
- if ( se==ss ) error("Could not parse: %s\n", tmp.s);
-
- sample2sex[ismpl] = ploidy_add_sex(ploidy, ss);
- }
- if ( hts_close(fp) ) error("Close failed: %s\n", fname);
- free(tmp.s);
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- int c, default_ploidy = 2;
- char *tags_str = "GT";
- char *ploidy_fname = NULL, *sex_fname = NULL;
-
- static struct option loptions[] =
- {
- {"default-ploidy",1,0,'d'},
- {"force-ploidy",1,0,'f'},
- {"ploidy",1,0,'p'},
- {"sex",1,0,'s'},
- {"tags",1,0,'t'},
- {0,0,0,0}
- };
- char *tmp;
- while ((c = getopt_long(argc, argv, "?ht:s:p:d:f:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'd':
- default_ploidy = strtod(optarg,&tmp);
- if ( *tmp ) error("Could not parse: -d %s\n", optarg);
- break;
- case 'f':
- force_ploidy = strtod(optarg,&tmp);
- if ( *tmp ) error("Could not parse: -f %s\n", optarg);
- break;
- case 'p': ploidy_fname = optarg; break;
- case 's': sex_fname = optarg; break;
- case 't': tags_str = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( strcasecmp("GT",tags_str) ) error("Only -t GT is currently supported, sorry\n");
-
- n_sample = bcf_hdr_nsamples(in);
- sample2sex = (int*) calloc(n_sample,sizeof(int));
- in_hdr = in;
- out_hdr = out;
-
- if ( ploidy_fname )
- ploidy = ploidy_init(ploidy_fname, default_ploidy);
- else if ( force_ploidy==-1 )
- {
- ploidy = ploidy_init_string(
- "X 1 60000 M 1\n"
- "X 2699521 154931043 M 1\n"
- "Y 1 59373566 M 1\n"
- "Y 1 59373566 F 0\n"
- "MT 1 16569 M 1\n"
- "MT 1 16569 F 1\n", 2);
- }
- if ( force_ploidy==-1 )
- {
- if ( !ploidy ) return -1;
-
- // add default sex in case it was not included
- int i, dflt_sex_id = ploidy_add_sex(ploidy, "F");
- for (i=0; i<n_sample; i++) sample2sex[i] = dflt_sex_id; // by default all are F
- if ( sex_fname ) set_samples(sex_fname, in, ploidy, sample2sex);
- nsex = ploidy_nsex(ploidy);
- sex2ploidy = (int*) malloc(sizeof(int)*nsex);
- }
-
- return 0;
-}
-
-
-bcf1_t *process(bcf1_t *rec)
-{
- int i,j, max_ploidy;
-
- int ngts = bcf_get_genotypes(in_hdr, rec, >_arr, &ngt_arr);
- if ( ngts<0 )
- return rec; // GT field not present
-
- if ( ngts % n_sample )
- error("Error at %s:%d: wrong number of GT fields\n",bcf_seqname(in_hdr,rec),rec->pos+1);
-
- if ( force_ploidy==-1 )
- ploidy_query(ploidy, (char*)bcf_seqname(in_hdr,rec), rec->pos, sex2ploidy,NULL,&max_ploidy);
- else
- max_ploidy = force_ploidy;
-
- ngts /= n_sample;
- if ( ngts < max_ploidy )
- {
- hts_expand(int32_t,max_ploidy*n_sample,ngt_arr2,gt_arr2);
- for (i=0; i<n_sample; i++)
- {
- int ploidy = force_ploidy!=-1 ? force_ploidy : sex2ploidy[ sample2sex[i] ];
- int32_t *src = >_arr[i*ngts];
- int32_t *dst = >_arr2[i*max_ploidy];
- j = 0;
- if ( !ploidy ) { dst[j] = bcf_gt_missing; j++; }
- else
- while ( j<ngts && j<ploidy && src[j]!=bcf_int32_vector_end ) { dst[j] = src[j]; j++; }
- assert( j );
- while ( j<ploidy ) { dst[j] = dst[j-1]; j++; } // expand "." to "./." and "0" to "0/0"
- while ( j<max_ploidy ) { dst[j] = bcf_int32_vector_end; j++; }
- }
- if ( bcf_update_genotypes(out_hdr,rec,gt_arr2,n_sample*max_ploidy) )
- error("Could not update GT field at %s:%d\n", bcf_seqname(in_hdr,rec),rec->pos+1);
- }
- else if ( ngts!=1 || max_ploidy!=1 )
- {
- for (i=0; i<n_sample; i++)
- {
- int ploidy = force_ploidy!=-1 ? force_ploidy : sex2ploidy[ sample2sex[i] ];
- int32_t *gts = >_arr[i*ngts];
- j = 0;
- if ( !ploidy ) { gts[j] = bcf_gt_missing; j++; }
- else
- while ( j<ngts && j<ploidy && gts[j]!=bcf_int32_vector_end ) j++;
- assert( j );
- while ( j<ploidy ) { gts[j] = gts[j-1]; j++; } // expand "." to "./." and "0" to "0/0"
- while ( j<ngts ) { gts[j] = bcf_int32_vector_end; j++; }
- }
- if ( bcf_update_genotypes(out_hdr,rec,gt_arr,n_sample*ngts) )
- error("Could not update GT field at %s:%d\n", bcf_seqname(in_hdr,rec),rec->pos+1);
- }
- return rec;
-}
-
-
-void destroy(void)
-{
- free(gt_arr);
- free(gt_arr2);
- free(sample2sex);
- free(sex2ploidy);
- if ( ploidy ) ploidy_destroy(ploidy);
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/*
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <ctype.h>
-#include <htslib/vcf.h>
-#include <htslib/kseq.h>
-#include "bcftools.h"
-#include "ploidy.h"
-
-static bcf_hdr_t *in_hdr = NULL, *out_hdr = NULL;
-static int *sample2sex = NULL;
-static int n_sample = 0, nsex = 0, *sex2ploidy = NULL;
-static int32_t ngt_arr = 0, *gt_arr = NULL, *gt_arr2 = NULL, ngt_arr2 = 0;
-static ploidy_t *ploidy = NULL;
-static int force_ploidy = -1;
-
-const char *about(void)
-{
- return "Fix ploidy.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Fix ploidy\n"
- "Usage: bcftools +fixploidy [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -d, --default-ploidy <int> default ploidy for regions unlisted in -p [2]\n"
- " -f, --force-ploidy <int> ignore -p, set the same ploidy for all genotypes\n"
- " -p, --ploidy <file> space/tab-delimited list of CHROM,FROM,TO,SEX,PLOIDY\n"
- " -s, --sex <file> list of samples, \"NAME SEX\"\n"
- " -t, --tags <list> VCF tags to fix [GT]\n"
- "\n"
- "Example:\n"
- " # Default ploidy, if -p not given. Unlisted regions have ploidy 2\n"
- " X 1 60000 M 1\n"
- " X 2699521 154931043 M 1\n"
- " Y 1 59373566 M 1\n"
- " Y 1 59373566 F 0\n"
- " MT 1 16569 M 1\n"
- " MT 1 16569 F 1\n"
- " \n"
- " # Example of -s file, sex of unlisted samples is \"F\"\n"
- " sampleName1 M\n"
- " \n"
- " bcftools +fixploidy in.vcf -- -s samples.txt\n"
- "\n";
-}
-
-void set_samples(char *fname, bcf_hdr_t *hdr, ploidy_t *ploidy, int *sample2sex)
-{
- kstring_t tmp = {0,0,0};
-
- htsFile *fp = hts_open(fname, "r");
- if ( !fp ) error("Could not read: %s\n", fname);
- while ( hts_getline(fp, KS_SEP_LINE, &tmp) > 0 )
- {
- char *ss = tmp.s;
- while ( *ss && isspace(*ss) ) ss++;
- if ( !*ss ) error("Could not parse: %s\n", tmp.s);
- if ( *ss=='#' ) continue;
- char *se = ss;
- while ( *se && !isspace(*se) ) se++;
- char x = *se; *se = 0;
-
- int ismpl = bcf_hdr_id2int(hdr, BCF_DT_SAMPLE, ss);
- if ( ismpl < 0 ) { fprintf(bcftools_stderr,"Warning: No such sample in the VCF: %s\n",ss); continue; }
-
- *se = x;
- ss = se+1;
- while ( *ss && isspace(*ss) ) ss++;
- if ( !*ss ) error("Could not parse: %s\n", tmp.s);
- se = ss;
- while ( *se && !isspace(*se) ) se++;
- if ( se==ss ) error("Could not parse: %s\n", tmp.s);
-
- sample2sex[ismpl] = ploidy_add_sex(ploidy, ss);
- }
- if ( hts_close(fp) ) error("Close failed: %s\n", fname);
- free(tmp.s);
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- int c, default_ploidy = 2;
- char *tags_str = "GT";
- char *ploidy_fname = NULL, *sex_fname = NULL;
-
- static struct option loptions[] =
- {
- {"default-ploidy",1,0,'d'},
- {"force-ploidy",1,0,'f'},
- {"ploidy",1,0,'p'},
- {"sex",1,0,'s'},
- {"tags",1,0,'t'},
- {0,0,0,0}
- };
- char *tmp;
- while ((c = getopt_long(argc, argv, "?ht:s:p:d:f:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'd':
- default_ploidy = strtod(optarg,&tmp);
- if ( *tmp ) error("Could not parse: -d %s\n", optarg);
- break;
- case 'f':
- force_ploidy = strtod(optarg,&tmp);
- if ( *tmp ) error("Could not parse: -f %s\n", optarg);
- break;
- case 'p': ploidy_fname = optarg; break;
- case 's': sex_fname = optarg; break;
- case 't': tags_str = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( strcasecmp("GT",tags_str) ) error("Only -t GT is currently supported, sorry\n");
-
- n_sample = bcf_hdr_nsamples(in);
- sample2sex = (int*) calloc(n_sample,sizeof(int));
- in_hdr = in;
- out_hdr = out;
-
- if ( ploidy_fname )
- ploidy = ploidy_init(ploidy_fname, default_ploidy);
- else if ( force_ploidy==-1 )
- {
- ploidy = ploidy_init_string(
- "X 1 60000 M 1\n"
- "X 2699521 154931043 M 1\n"
- "Y 1 59373566 M 1\n"
- "Y 1 59373566 F 0\n"
- "MT 1 16569 M 1\n"
- "MT 1 16569 F 1\n", 2);
- }
- if ( force_ploidy==-1 )
- {
- if ( !ploidy ) return -1;
-
- // add default sex in case it was not included
- int i, dflt_sex_id = ploidy_add_sex(ploidy, "F");
- for (i=0; i<n_sample; i++) sample2sex[i] = dflt_sex_id; // by default all are F
- if ( sex_fname ) set_samples(sex_fname, in, ploidy, sample2sex);
- nsex = ploidy_nsex(ploidy);
- sex2ploidy = (int*) malloc(sizeof(int)*nsex);
- }
-
- return 0;
-}
-
-
-bcf1_t *process(bcf1_t *rec)
-{
- int i,j, max_ploidy;
-
- int ngts = bcf_get_genotypes(in_hdr, rec, >_arr, &ngt_arr);
- if ( ngts<0 )
- return rec; // GT field not present
-
- if ( ngts % n_sample )
- error("Error at %s:%d: wrong number of GT fields\n",bcf_seqname(in_hdr,rec),rec->pos+1);
-
- if ( force_ploidy==-1 )
- ploidy_query(ploidy, (char*)bcf_seqname(in_hdr,rec), rec->pos, sex2ploidy,NULL,&max_ploidy);
- else
- max_ploidy = force_ploidy;
-
- ngts /= n_sample;
- if ( ngts < max_ploidy )
- {
- hts_expand(int32_t,max_ploidy*n_sample,ngt_arr2,gt_arr2);
- for (i=0; i<n_sample; i++)
- {
- int ploidy = force_ploidy!=-1 ? force_ploidy : sex2ploidy[ sample2sex[i] ];
- int32_t *src = >_arr[i*ngts];
- int32_t *dst = >_arr2[i*max_ploidy];
- j = 0;
- if ( !ploidy ) { dst[j] = bcf_gt_missing; j++; }
- else
- while ( j<ngts && j<ploidy && src[j]!=bcf_int32_vector_end ) { dst[j] = src[j]; j++; }
- assert( j );
- while ( j<ploidy ) { dst[j] = dst[j-1]; j++; } // expand "." to "./." and "0" to "0/0"
- while ( j<max_ploidy ) { dst[j] = bcf_int32_vector_end; j++; }
- }
- if ( bcf_update_genotypes(out_hdr,rec,gt_arr2,n_sample*max_ploidy) )
- error("Could not update GT field at %s:%d\n", bcf_seqname(in_hdr,rec),rec->pos+1);
- }
- else if ( ngts!=1 || max_ploidy!=1 )
- {
- for (i=0; i<n_sample; i++)
- {
- int ploidy = force_ploidy!=-1 ? force_ploidy : sex2ploidy[ sample2sex[i] ];
- int32_t *gts = >_arr[i*ngts];
- j = 0;
- if ( !ploidy ) { gts[j] = bcf_gt_missing; j++; }
- else
- while ( j<ngts && j<ploidy && gts[j]!=bcf_int32_vector_end ) j++;
- assert( j );
- while ( j<ploidy ) { gts[j] = gts[j-1]; j++; } // expand "." to "./." and "0" to "0/0"
- while ( j<ngts ) { gts[j] = bcf_int32_vector_end; j++; }
- }
- if ( bcf_update_genotypes(out_hdr,rec,gt_arr,n_sample*ngts) )
- error("Could not update GT field at %s:%d\n", bcf_seqname(in_hdr,rec),rec->pos+1);
- }
- return rec;
-}
-
-
-void destroy(void)
-{
- free(gt_arr);
- free(gt_arr2);
- free(sample2sex);
- free(sex2ploidy);
- if ( ploidy ) ploidy_destroy(ploidy);
-}
-
-
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-/*
- Illumina TOP/BOT strand convention causes a lot of pain. This tool
- attempts to determine the strand convention and convert it to the
- forward reference strand.
-
- On TOP strand, we can encounter
- unambiguous SNPs:
- A/G
- A/C
- ambiguous (context-dependent) SNPs:
- C/G
- A/T
-
- On BOT strand:
- unambiguous SNPs:
- T/G
- T/C
- ambiguous (context-dependent) SNPs:
- T/A
- G/C
-
-
- For unambiguous pairs (A/C, A/G, T/C, T/G), the knowledge of reference base
- at the SNP position is enough to determine the strand:
-
- TOP REF -> ALLELES TOP_ON_STRAND
- -------------------------------------------
- A/C A or C A/C 1
- " T or G T/G -1
- A/G A or G A/G 1
- " T or C T/C -1
-
-
- For ambiguous pairs (A/T, C/G), a sequence walking must be performed
- (simultaneously upstream and downstream) until the first unambiguous pair
- is encountered. The 5' base determines the strand:
-
- TOP 5'REF_BASE -> ALLELES TOP_ON_STRAND
- ------------------------------------------------
- A/T A or T A/T 1
- " C or G T/A -1
- C/G A or T C/G 1
- " C or G G/C -1
-
- */
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/kfunc.h>
-#include <htslib/faidx.h>
-#include <htslib/khash.h>
-#include <htslib/synced_bcf_reader.h>
-#include "bcftools.h"
-
-#define MODE_STATS 1
-#define MODE_TOP2FWD 2
-#define MODE_FLIP2FWD 3
-#define MODE_USE_ID 4
-
-typedef struct
-{
- uint32_t pos;
- uint8_t ref;
-}
-marker_t;
-
-KHASH_MAP_INIT_INT(i2m, marker_t)
-typedef khash_t(i2m) i2m_t;
-
-typedef struct
-{
- char *dbsnp_fname;
- int mode, discard;
- bcf_hdr_t *hdr;
- faidx_t *fai;
- int rid, skip_rid;
- i2m_t *i2m;
- int32_t *gts, ngts, pos;
- uint32_t nsite,nok,nflip,nunresolved,nswap,nflip_swap,nonSNP,nonACGT,nonbiallelic;
- uint32_t count[4][4], npos_err, unsorted;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Fix reference strand orientation, e.g. from Illumina/TOP to fwd.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: This tool helps to determine and fix strand orientation.\n"
- " Currently the following modes are recognised:\n"
- " flip .. flips non-ambiguous SNPs and ignores the rest\n"
- " id .. swap REF/ALT and GTs using the ID column to determine the REF allele\n"
- " stats .. collect and print stats\n"
- " top .. converts from Illumina TOP strand to fwd\n"
- "\n"
- " WARNING: Do not use the program blindly, make an effort to\n"
- " understand what strand convention your data uses! Make sure\n"
- " the reason for mismatching REF alleles is not a different\n"
- " reference build!!\n"
- "\n"
- "Usage: bcftools +fixref [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -d, --discard Discard sites which could not be resolved\n"
- " -f, --fasta-ref <file.fa> Reference sequence\n"
- " -i, --use-id <file.vcf> Swap REF/ALT using the ID column to determine the REF allele, implies -m id.\n"
- " Download the dbSNP file from\n"
- " https://www.ncbi.nlm.nih.gov/variation/docs/human_variation_vcf\n"
- " -m, --mode <string> Collect stats (\"stats\") or convert (\"flip\", \"id\", \"top\") [stats]\n"
- "\n"
- "Examples:\n"
- " # run stats\n"
- " bcftools +fixref file.bcf -- -f ref.fa\n"
- "\n"
- " # convert from TOP to fwd\n"
- " bcftools +fixref file.bcf -Ob -o out.bcf -- -f ref.fa -m top\n"
- "\n"
- " # match the REF/ALT alleles based on the ID column, discard unknown sites\n"
- " bcftools +fixref file.bcf -Ob -o out.bcf -- -d -f ref.fa -i All_20151104.vcf.gz\n"
- "\n"
- " # assuming the reference build is correct, just flip to fwd, discarding the rest\n"
- " bcftools +fixref file.bcf -Ob -o out.bcf -- -d -f ref.fa -m flip\n"
- "\n";
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- memset(&args,0,sizeof(args_t));
- args.skip_rid = -1;
- args.hdr = in;
- args.mode = MODE_STATS;
- char *ref_fname = NULL;
- static struct option loptions[] =
- {
- {"mode",required_argument,NULL,'m'},
- {"discard",no_argument,NULL,'d'},
- {"fasta-ref",required_argument,NULL,'f'},
- {"use-id",required_argument,NULL,'i'},
- {NULL,0,NULL,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?hf:m:di:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'm':
- if ( !strcasecmp(optarg,"top") ) args.mode = MODE_TOP2FWD;
- else if ( !strcasecmp(optarg,"flip") ) args.mode = MODE_FLIP2FWD;
- else if ( !strcasecmp(optarg,"id") ) args.mode = MODE_USE_ID;
- else if ( !strcasecmp(optarg,"stats") ) args.mode = MODE_STATS;
- else error("The source strand convention not recognised: %s\n", optarg);
- break;
- case 'i': args.dbsnp_fname = optarg; args.mode = MODE_USE_ID; break;
- case 'd': args.discard = 1; break;
- case 'f': ref_fname = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( !ref_fname ) error("Expected the -f option\n");
- args.fai = fai_load(ref_fname);
- if ( !args.fai ) error("Failed to load the fai index: %s\n", ref_fname);
-
- if ( args.mode==MODE_STATS ) return 1;
- return 0;
-}
-
-static bcf1_t *set_ref_alt(args_t *args, bcf1_t *rec, const char ref, const char alt, int swap)
-{
- rec->d.allele[0][0] = ref;
- rec->d.allele[1][0] = alt;
- rec->d.shared_dirty |= BCF1_DIRTY_ALS;
-
- if ( !swap ) return rec; // only fix the alleles, leaving GTs unchanged
-
- int ngts = bcf_get_genotypes(args->hdr, rec, &args->gts, &args->ngts);
- int i, j, nsmpl = bcf_hdr_nsamples(args->hdr);
- ngts /= nsmpl;
- for (i=0; i<nsmpl; i++)
- {
- int32_t *ptr = args->gts + i*ngts;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_gt_unphased(0) ) ptr[j] = bcf_gt_unphased(1);
- else if ( ptr[j]==bcf_gt_phased(0) ) ptr[j] = bcf_gt_phased(1);
- else if ( ptr[j]==bcf_gt_unphased(1) ) ptr[j] = bcf_gt_unphased(0);
- else if ( ptr[j]==bcf_gt_phased(1) ) ptr[j] = bcf_gt_phased(0);
- }
- }
- bcf_update_genotypes(args->hdr,rec,args->gts,args->ngts);
-
- return rec;
-}
-
-static inline int nt2int(char nt)
-{
- nt = toupper(nt);
- if ( nt=='A' ) return 0;
- if ( nt=='C' ) return 1;
- if ( nt=='G' ) return 2;
- if ( nt=='T' ) return 3;
- return -1;
-}
-#define int2nt(x) "ACGT"[x]
-#define revint(x) ("3210"[x]-'0')
-
-static inline uint32_t parse_rsid(char *name)
-{
- if ( name[0]!='r' || name[1]!='s' )
- {
- name = strstr(name, "rs");
- if ( !name ) return 0;
- }
- char *tmp;
- name += 2;
- uint64_t id = strtol(name, &tmp, 10);
- if ( tmp==name || *tmp ) return 0;
- if ( id > UINT32_MAX ) error("FIXME: the ID is too big for uint32_t: %s\n", name-2);
- return id;
-}
-
-static int fetch_ref(args_t *args, bcf1_t *rec)
-{
- // Get the reference allele
- int len;
- char *ref = faidx_fetch_seq(args->fai, (char*)bcf_seqname(args->hdr,rec), rec->pos, rec->pos, &len);
- if ( !ref )
- {
- if ( faidx_has_seq(args->fai, bcf_seqname(args->hdr,rec))==0 )
- {
- fprintf(stderr,"Ignoring sequence \"%s\"\n", bcf_seqname(args->hdr,rec));
- args->skip_rid = rec->rid;
- return -2;
- }
- error("faidx_fetch_seq failed at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
- }
- int ir = nt2int(*ref);
- free(ref);
- return ir;
-}
-
-static void dbsnp_init(args_t *args, const char *chr)
-{
- if ( args->i2m ) kh_destroy(i2m, args->i2m);
- args->i2m = kh_init(i2m);
- bcf_srs_t *sr = bcf_sr_init();
- if ( bcf_sr_set_regions(sr, chr, 0) != 0 ) goto done;
- if ( !bcf_sr_add_reader(sr,args->dbsnp_fname) ) error("Failed to open %s: %s\n", args->dbsnp_fname,bcf_sr_strerror(sr->errnum));
- while ( bcf_sr_next_line(sr) )
- {
- bcf1_t *rec = bcf_sr_get_line(sr, 0);
- if ( rec->d.allele[0][1]!=0 || rec->d.allele[1][1]!=0 ) continue; // skip non-snps
-
- int ref = nt2int(rec->d.allele[0][0]);
- if ( ref<0 ) continue; // non-[ACGT] base
-
- uint32_t id = parse_rsid(rec->d.id);
- if ( !id ) continue;
-
- int ret, k;
- k = kh_put(i2m, args->i2m, id, &ret);
- if ( ret<0 ) error("An error occurred while inserting the key %u\n", id);
- if ( ret==0 ) continue; // skip ambiguous id
- kh_val(args->i2m, k).pos = (uint32_t)rec->pos;
- kh_val(args->i2m, k).ref = ref;
- }
-done:
- bcf_sr_destroy(sr);
-}
-
-static bcf1_t *dbsnp_check(args_t *args, bcf1_t *rec, int ir, int ia, int ib)
-{
- int k, ref,pos;
- uint32_t id = parse_rsid(rec->d.id);
- if ( !id ) goto no_info;
-
- k = kh_get(i2m, args->i2m, id);
- if ( k==kh_end(args->i2m) ) goto no_info;
-
- pos = (int)kh_val(args->i2m, k).pos;
- if ( pos != rec->pos )
- {
- rec->pos = pos;
- ir = fetch_ref(args, rec);
- args->npos_err++;
- }
-
- ref = kh_val(args->i2m, k).ref;
- if ( ref!=ir )
- error("Reference base mismatch at %s:%d .. %c vs %c\n",bcf_seqname(args->hdr,rec),rec->pos+1,int2nt(ref),int2nt(ir));
-
- if ( ia==ref ) return rec;
- if ( ib==ref ) { args->nswap++; return set_ref_alt(args,rec,int2nt(ib),int2nt(ia),1); }
-
-no_info:
- args->nunresolved++;
- return args->discard ? NULL : rec;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- if ( rec->rid == args.skip_rid ) return NULL;
-
- bcf1_t *ret = args.mode==MODE_STATS ? NULL : rec;
- args.nsite++;
-
- // Skip non-SNPs
- if ( bcf_get_variant_types(rec)!=VCF_SNP )
- {
- args.nonSNP++;
- return args.discard ? NULL : ret;
- }
-
- // Get the reference allele
- int ir = fetch_ref(&args, rec);
- if ( ir==-2 ) return NULL;
- if ( ir==-1 )
- {
- args.nonACGT++;
- return args.discard ? NULL : ret; // not A,C,G,T
- }
-
- if ( rec->n_allele!=2 )
- {
- // not a biallelic site
- args.nonbiallelic++;
- return args.discard ? NULL : ret;
- }
-
- int ia = nt2int(rec->d.allele[0][0]);
- if ( ia<0 )
- {
- // not A,C,G,T
- args.nonACGT++;
- return args.discard ? NULL : ret;
- }
-
- int ib = nt2int(rec->d.allele[1][0]);
- if ( ib<0 )
- {
- // not A,C,G,T
- args.nonACGT++;
- return args.discard ? NULL : ret;
- }
-
- if ( ia==ib )
- {
- // should not happen in well-formed VCF
- args.nonSNP++;
- return args.discard ? NULL : ret;
- }
- args.count[ia][ib]++;
-
- if ( ir==ia ) args.nok++;
-
- if ( args.mode==MODE_USE_ID )
- {
- if ( !args.i2m || args.rid!=rec->rid )
- {
- args.pos = 0;
- args.rid = rec->rid;
- dbsnp_init(&args,bcf_seqname(args.hdr,rec));
- }
- ret = dbsnp_check(&args, rec, ir,ia,ib);
- if ( !args.unsorted && args.pos > rec->pos )
- {
- fprintf(stderr,
- "Warning: corrected position(s) results in unsorted VCF, for example %s:%d comes after %s:%d\n"
- " The standard unix `sort` or `vcf-sort` from vcftools can be used to fix the order.\n",
- bcf_seqname(args.hdr,rec),rec->pos+1,bcf_seqname(args.hdr,rec),args.pos);
- args.unsorted = 1;
- }
- args.pos = rec->pos;
- return ret;
- }
- else if ( args.mode==MODE_FLIP2FWD )
- {
- int pair = 1 << ia | 1 << ib;
- if ( pair==0x9 || pair==0x6 ) // skip ambiguous pairs: A/T or C/G
- {
- args.nunresolved++;
- return args.discard ? NULL : ret;
- }
- if ( ir==ia ) return ret;
- if ( ir==ib ) { args.nswap++; return set_ref_alt(&args,rec,int2nt(ib),int2nt(ia),1); }
- if ( ir==revint(ia) ) { args.nflip++; return set_ref_alt(&args,rec,int2nt(revint(ia)),int2nt(revint(ib)),0); }
- if ( ir==revint(ib) ) { args.nflip_swap++; return set_ref_alt(&args,rec,int2nt(revint(ib)),int2nt(revint(ia)),1); }
- error("FIXME: this should not happen %s:%d\n", bcf_seqname(args.hdr,rec),rec->pos+1);
- }
- else if ( args.mode==MODE_TOP2FWD )
- {
- int pair = 1 << ia | 1 << ib;
- if ( pair != 0x9 && pair != 0x6 ) // unambiguous pair: A/C or A/G
- {
- if ( ir==ia ) return ret;
-
- int ia_rev = revint(ia);
- if ( ir==ia_rev ) // vcfref is A, faref is T, flip
- {
- args.nflip++;
- return set_ref_alt(&args,rec,int2nt(ia_rev),int2nt(revint(ib)),0);
- }
- if ( ir==ib ) // vcfalt is faref, swap
- {
- args.nswap++;
- return set_ref_alt(&args,rec,int2nt(ib),int2nt(ia),1);
- }
- assert( ib==revint(ir) );
-
- args.nflip_swap++;
- return set_ref_alt(&args,rec,int2nt(revint(ib)),int2nt(revint(ia)),1);
- }
- else // ambiguous pair, sequence walking must be performed
- {
- int len, win = rec->pos > 100 ? 100 : rec->pos, beg = rec->pos - win, end = rec->pos + win;
- char *ref = faidx_fetch_seq(args.fai, (char*)bcf_seqname(args.hdr,rec), beg,end, &len);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", bcf_seqname(args.hdr,rec),rec->pos+1);
- if ( end - beg + 1 != len ) error("FIXME: check win=%d,len=%d at %s:%d (%d %d)\n", win,len, bcf_seqname(args.hdr,rec),rec->pos+1, end,beg);
-
- int i, mid = rec->pos - beg, strand = 0;
- for (i=1; i<=win; i++)
- {
- int ra = nt2int(ref[mid-i]);
- int rb = nt2int(ref[mid+i]);
- if ( ra<0 || rb<0 || ra==rb ) continue; // skip N's and non-infomative pairs: A/A, C/C, G/G, T/T
- pair = 1 << ra | 1 << rb;
- if ( pair==0x9 || pair==0x6 ) continue; // skip ambiguous pairs: A/T or C/G
- strand = 1 << ra & 0x9 ? 1 : -1;
- break;
- }
- free(ref);
-
- if ( strand==1 )
- {
- if ( ir==ia ) return ret;
- if ( ir==ib )
- {
- args.nswap++;
- return set_ref_alt(&args,rec,int2nt(ib),int2nt(ia),1);
- }
- }
- else if ( strand==-1 )
- {
- int ia_rev = revint(ia);
- int ib_rev = revint(ib);
- if ( ir==ia_rev )
- {
- args.nflip++;
- return set_ref_alt(&args,rec,int2nt(ia_rev),int2nt(ib_rev),0);
- }
- if ( ir==ib_rev )
- {
- args.nflip_swap++;
- return set_ref_alt(&args,rec,int2nt(ib_rev),int2nt(ia_rev),1);
- }
- }
-
- args.nunresolved++;
- return args.discard ? NULL : ret;
- }
- }
- return ret;
-}
-
-int top_mask[4][4] =
-{
- {0,1,1,1},
- {0,0,1,0},
- {0,0,0,0},
- {0,0,0,0},
-};
-int bot_mask[4][4] =
-{
- {0,0,0,0},
- {0,0,0,0},
- {0,1,0,0},
- {1,1,1,0},
-};
-
-void destroy(void)
-{
- uint32_t i,j,tot = 0;
- uint32_t top_err = 0, bot_err = 0;
- for (i=0; i<4; i++)
- {
- for (j=0; j<4; j++)
- {
- tot += args.count[i][j];
- if ( !top_mask[i][j] && args.count[i][j] ) top_err++;
- if ( !bot_mask[i][j] && args.count[i][j] ) bot_err++;
- }
- }
- uint32_t nskip = args.nonACGT+args.nonSNP+args.nonbiallelic;
- uint32_t ncmp = args.nsite - nskip;
-
- fprintf(stderr,"# SC, guessed strand convention\n");
- fprintf(stderr,"SC\tTOP-compatible\t%d\n",top_err?0:1);
- fprintf(stderr,"SC\tBOT-compatible\t%d\n",bot_err?0:1);
-
- fprintf(stderr,"# ST, substitution types\n");
- for (i=0; i<4; i++)
- {
- for (j=0; j<4; j++)
- {
- if ( i==j ) continue;
- fprintf(stderr,"ST\t%c>%c\t%u\t%.1f%%\n", int2nt(i),int2nt(j),args.count[i][j], args.count[i][j]*100./tot);
- }
- }
- fprintf(stderr,"# NS, Number of sites:\n");
- fprintf(stderr,"NS\ttotal \t%u\n", args.nsite);
- fprintf(stderr,"NS\tref match \t%u\t%.1f%%\n", args.nok,100.*args.nok/ncmp);
- fprintf(stderr,"NS\tref mismatch \t%u\t%.1f%%\n", ncmp-args.nok,100.*(ncmp-args.nok)/ncmp);
- if ( args.mode!=MODE_STATS )
- {
- fprintf(stderr,"NS\tflipped \t%u\t%.1f%%\n", args.nflip,100.*args.nflip/(args.nsite-nskip));
- fprintf(stderr,"NS\tswapped \t%u\t%.1f%%\n", args.nswap,100.*args.nswap/(args.nsite-nskip));
- fprintf(stderr,"NS\tflip+swap \t%u\t%.1f%%\n", args.nflip_swap,100.*args.nflip_swap/(args.nsite-nskip));
- fprintf(stderr,"NS\tunresolved \t%u\t%.1f%%\n", args.nunresolved,100.*args.nunresolved/(args.nsite-nskip));
- fprintf(stderr,"NS\tfixed pos \t%u\t%.1f%%\n", args.npos_err,100.*args.npos_err/(args.nsite-nskip));
- }
- fprintf(stderr,"NS\tskipped \t%u\n", nskip);
- fprintf(stderr,"NS\tnon-ACGT \t%u\n", args.nonACGT);
- fprintf(stderr,"NS\tnon-SNP \t%u\n", args.nonSNP);
- fprintf(stderr,"NS\tnon-biallelic\t%u\n", args.nonbiallelic);
-
- free(args.gts);
- if ( args.fai ) fai_destroy(args.fai);
- if ( args.i2m ) kh_destroy(i2m, args.i2m);
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-/*
- Illumina TOP/BOT strand convention causes a lot of pain. This tool
- attempts to determine the strand convention and convert it to the
- forward reference strand.
-
- On TOP strand, we can encounter
- unambiguous SNPs:
- A/G
- A/C
- ambiguous (context-dependent) SNPs:
- C/G
- A/T
-
- On BOT strand:
- unambiguous SNPs:
- T/G
- T/C
- ambiguous (context-dependent) SNPs:
- T/A
- G/C
-
-
- For unambiguous pairs (A/C, A/G, T/C, T/G), the knowledge of reference base
- at the SNP position is enough to determine the strand:
-
- TOP REF -> ALLELES TOP_ON_STRAND
- -------------------------------------------
- A/C A or C A/C 1
- " T or G T/G -1
- A/G A or G A/G 1
- " T or C T/C -1
-
-
- For ambiguous pairs (A/T, C/G), a sequence walking must be performed
- (simultaneously upstream and downstream) until the first unambiguous pair
- is encountered. The 5' base determines the strand:
-
- TOP 5'REF_BASE -> ALLELES TOP_ON_STRAND
- ------------------------------------------------
- A/T A or T A/T 1
- " C or G T/A -1
- C/G A or T C/G 1
- " C or G G/C -1
-
- */
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/kfunc.h>
-#include <htslib/faidx.h>
-#include <htslib/khash.h>
-#include <htslib/synced_bcf_reader.h>
-#include "bcftools.h"
-
-#define MODE_STATS 1
-#define MODE_TOP2FWD 2
-#define MODE_FLIP2FWD 3
-#define MODE_USE_ID 4
-
-typedef struct
-{
- uint32_t pos;
- uint8_t ref;
-}
-marker_t;
-
-KHASH_MAP_INIT_INT(i2m, marker_t)
-typedef khash_t(i2m) i2m_t;
-
-typedef struct
-{
- char *dbsnp_fname;
- int mode, discard;
- bcf_hdr_t *hdr;
- faidx_t *fai;
- int rid, skip_rid;
- i2m_t *i2m;
- int32_t *gts, ngts, pos;
- uint32_t nsite,nok,nflip,nunresolved,nswap,nflip_swap,nonSNP,nonACGT,nonbiallelic;
- uint32_t count[4][4], npos_err, unsorted;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Fix reference strand orientation, e.g. from Illumina/TOP to fwd.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: This tool helps to determine and fix strand orientation.\n"
- " Currently the following modes are recognised:\n"
- " flip .. flips non-ambiguous SNPs and ignores the rest\n"
- " id .. swap REF/ALT and GTs using the ID column to determine the REF allele\n"
- " stats .. collect and print stats\n"
- " top .. converts from Illumina TOP strand to fwd\n"
- "\n"
- " WARNING: Do not use the program blindly, make an effort to\n"
- " understand what strand convention your data uses! Make sure\n"
- " the reason for mismatching REF alleles is not a different\n"
- " reference build!!\n"
- "\n"
- "Usage: bcftools +fixref [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -d, --discard Discard sites which could not be resolved\n"
- " -f, --fasta-ref <file.fa> Reference sequence\n"
- " -i, --use-id <file.vcf> Swap REF/ALT using the ID column to determine the REF allele, implies -m id.\n"
- " Download the dbSNP file from\n"
- " https://www.ncbi.nlm.nih.gov/variation/docs/human_variation_vcf\n"
- " -m, --mode <string> Collect stats (\"stats\") or convert (\"flip\", \"id\", \"top\") [stats]\n"
- "\n"
- "Examples:\n"
- " # run stats\n"
- " bcftools +fixref file.bcf -- -f ref.fa\n"
- "\n"
- " # convert from TOP to fwd\n"
- " bcftools +fixref file.bcf -Ob -o out.bcf -- -f ref.fa -m top\n"
- "\n"
- " # match the REF/ALT alleles based on the ID column, discard unknown sites\n"
- " bcftools +fixref file.bcf -Ob -o out.bcf -- -d -f ref.fa -i All_20151104.vcf.gz\n"
- "\n"
- " # assuming the reference build is correct, just flip to fwd, discarding the rest\n"
- " bcftools +fixref file.bcf -Ob -o out.bcf -- -d -f ref.fa -m flip\n"
- "\n";
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- memset(&args,0,sizeof(args_t));
- args.skip_rid = -1;
- args.hdr = in;
- args.mode = MODE_STATS;
- char *ref_fname = NULL;
- static struct option loptions[] =
- {
- {"mode",required_argument,NULL,'m'},
- {"discard",no_argument,NULL,'d'},
- {"fasta-ref",required_argument,NULL,'f'},
- {"use-id",required_argument,NULL,'i'},
- {NULL,0,NULL,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?hf:m:di:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'm':
- if ( !strcasecmp(optarg,"top") ) args.mode = MODE_TOP2FWD;
- else if ( !strcasecmp(optarg,"flip") ) args.mode = MODE_FLIP2FWD;
- else if ( !strcasecmp(optarg,"id") ) args.mode = MODE_USE_ID;
- else if ( !strcasecmp(optarg,"stats") ) args.mode = MODE_STATS;
- else error("The source strand convention not recognised: %s\n", optarg);
- break;
- case 'i': args.dbsnp_fname = optarg; args.mode = MODE_USE_ID; break;
- case 'd': args.discard = 1; break;
- case 'f': ref_fname = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( !ref_fname ) error("Expected the -f option\n");
- args.fai = fai_load(ref_fname);
- if ( !args.fai ) error("Failed to load the fai index: %s\n", ref_fname);
-
- if ( args.mode==MODE_STATS ) return 1;
- return 0;
-}
-
-static bcf1_t *set_ref_alt(args_t *args, bcf1_t *rec, const char ref, const char alt, int swap)
-{
- rec->d.allele[0][0] = ref;
- rec->d.allele[1][0] = alt;
- rec->d.shared_dirty |= BCF1_DIRTY_ALS;
-
- if ( !swap ) return rec; // only fix the alleles, leaving GTs unchanged
-
- int ngts = bcf_get_genotypes(args->hdr, rec, &args->gts, &args->ngts);
- int i, j, nsmpl = bcf_hdr_nsamples(args->hdr);
- ngts /= nsmpl;
- for (i=0; i<nsmpl; i++)
- {
- int32_t *ptr = args->gts + i*ngts;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_gt_unphased(0) ) ptr[j] = bcf_gt_unphased(1);
- else if ( ptr[j]==bcf_gt_phased(0) ) ptr[j] = bcf_gt_phased(1);
- else if ( ptr[j]==bcf_gt_unphased(1) ) ptr[j] = bcf_gt_unphased(0);
- else if ( ptr[j]==bcf_gt_phased(1) ) ptr[j] = bcf_gt_phased(0);
- }
- }
- bcf_update_genotypes(args->hdr,rec,args->gts,args->ngts);
-
- return rec;
-}
-
-static inline int nt2int(char nt)
-{
- nt = toupper(nt);
- if ( nt=='A' ) return 0;
- if ( nt=='C' ) return 1;
- if ( nt=='G' ) return 2;
- if ( nt=='T' ) return 3;
- return -1;
-}
-#define int2nt(x) "ACGT"[x]
-#define revint(x) ("3210"[x]-'0')
-
-static inline uint32_t parse_rsid(char *name)
-{
- if ( name[0]!='r' || name[1]!='s' )
- {
- name = strstr(name, "rs");
- if ( !name ) return 0;
- }
- char *tmp;
- name += 2;
- uint64_t id = strtol(name, &tmp, 10);
- if ( tmp==name || *tmp ) return 0;
- if ( id > UINT32_MAX ) error("FIXME: the ID is too big for uint32_t: %s\n", name-2);
- return id;
-}
-
-static int fetch_ref(args_t *args, bcf1_t *rec)
-{
- // Get the reference allele
- int len;
- char *ref = faidx_fetch_seq(args->fai, (char*)bcf_seqname(args->hdr,rec), rec->pos, rec->pos, &len);
- if ( !ref )
- {
- if ( faidx_has_seq(args->fai, bcf_seqname(args->hdr,rec))==0 )
- {
- fprintf(bcftools_stderr,"Ignoring sequence \"%s\"\n", bcf_seqname(args->hdr,rec));
- args->skip_rid = rec->rid;
- return -2;
- }
- error("faidx_fetch_seq failed at %s:%d\n", bcf_seqname(args->hdr,rec),rec->pos+1);
- }
- int ir = nt2int(*ref);
- free(ref);
- return ir;
-}
-
-static void dbsnp_init(args_t *args, const char *chr)
-{
- if ( args->i2m ) kh_destroy(i2m, args->i2m);
- args->i2m = kh_init(i2m);
- bcf_srs_t *sr = bcf_sr_init();
- if ( bcf_sr_set_regions(sr, chr, 0) != 0 ) goto done;
- if ( !bcf_sr_add_reader(sr,args->dbsnp_fname) ) error("Failed to open %s: %s\n", args->dbsnp_fname,bcf_sr_strerror(sr->errnum));
- while ( bcf_sr_next_line(sr) )
- {
- bcf1_t *rec = bcf_sr_get_line(sr, 0);
- if ( rec->d.allele[0][1]!=0 || rec->d.allele[1][1]!=0 ) continue; // skip non-snps
-
- int ref = nt2int(rec->d.allele[0][0]);
- if ( ref<0 ) continue; // non-[ACGT] base
-
- uint32_t id = parse_rsid(rec->d.id);
- if ( !id ) continue;
-
- int ret, k;
- k = kh_put(i2m, args->i2m, id, &ret);
- if ( ret<0 ) error("An error occurred while inserting the key %u\n", id);
- if ( ret==0 ) continue; // skip ambiguous id
- kh_val(args->i2m, k).pos = (uint32_t)rec->pos;
- kh_val(args->i2m, k).ref = ref;
- }
-done:
- bcf_sr_destroy(sr);
-}
-
-static bcf1_t *dbsnp_check(args_t *args, bcf1_t *rec, int ir, int ia, int ib)
-{
- int k, ref,pos;
- uint32_t id = parse_rsid(rec->d.id);
- if ( !id ) goto no_info;
-
- k = kh_get(i2m, args->i2m, id);
- if ( k==kh_end(args->i2m) ) goto no_info;
-
- pos = (int)kh_val(args->i2m, k).pos;
- if ( pos != rec->pos )
- {
- rec->pos = pos;
- ir = fetch_ref(args, rec);
- args->npos_err++;
- }
-
- ref = kh_val(args->i2m, k).ref;
- if ( ref!=ir )
- error("Reference base mismatch at %s:%d .. %c vs %c\n",bcf_seqname(args->hdr,rec),rec->pos+1,int2nt(ref),int2nt(ir));
-
- if ( ia==ref ) return rec;
- if ( ib==ref ) { args->nswap++; return set_ref_alt(args,rec,int2nt(ib),int2nt(ia),1); }
-
-no_info:
- args->nunresolved++;
- return args->discard ? NULL : rec;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- if ( rec->rid == args.skip_rid ) return NULL;
-
- bcf1_t *ret = args.mode==MODE_STATS ? NULL : rec;
- args.nsite++;
-
- // Skip non-SNPs
- if ( bcf_get_variant_types(rec)!=VCF_SNP )
- {
- args.nonSNP++;
- return args.discard ? NULL : ret;
- }
-
- // Get the reference allele
- int ir = fetch_ref(&args, rec);
- if ( ir==-2 ) return NULL;
- if ( ir==-1 )
- {
- args.nonACGT++;
- return args.discard ? NULL : ret; // not A,C,G,T
- }
-
- if ( rec->n_allele!=2 )
- {
- // not a biallelic site
- args.nonbiallelic++;
- return args.discard ? NULL : ret;
- }
-
- int ia = nt2int(rec->d.allele[0][0]);
- if ( ia<0 )
- {
- // not A,C,G,T
- args.nonACGT++;
- return args.discard ? NULL : ret;
- }
-
- int ib = nt2int(rec->d.allele[1][0]);
- if ( ib<0 )
- {
- // not A,C,G,T
- args.nonACGT++;
- return args.discard ? NULL : ret;
- }
-
- if ( ia==ib )
- {
- // should not happen in well-formed VCF
- args.nonSNP++;
- return args.discard ? NULL : ret;
- }
- args.count[ia][ib]++;
-
- if ( ir==ia ) args.nok++;
-
- if ( args.mode==MODE_USE_ID )
- {
- if ( !args.i2m || args.rid!=rec->rid )
- {
- args.pos = 0;
- args.rid = rec->rid;
- dbsnp_init(&args,bcf_seqname(args.hdr,rec));
- }
- ret = dbsnp_check(&args, rec, ir,ia,ib);
- if ( !args.unsorted && args.pos > rec->pos )
- {
- fprintf(bcftools_stderr,
- "Warning: corrected position(s) results in unsorted VCF, for example %s:%d comes after %s:%d\n"
- " The standard unix `sort` or `vcf-sort` from vcftools can be used to fix the order.\n",
- bcf_seqname(args.hdr,rec),rec->pos+1,bcf_seqname(args.hdr,rec),args.pos);
- args.unsorted = 1;
- }
- args.pos = rec->pos;
- return ret;
- }
- else if ( args.mode==MODE_FLIP2FWD )
- {
- int pair = 1 << ia | 1 << ib;
- if ( pair==0x9 || pair==0x6 ) // skip ambiguous pairs: A/T or C/G
- {
- args.nunresolved++;
- return args.discard ? NULL : ret;
- }
- if ( ir==ia ) return ret;
- if ( ir==ib ) { args.nswap++; return set_ref_alt(&args,rec,int2nt(ib),int2nt(ia),1); }
- if ( ir==revint(ia) ) { args.nflip++; return set_ref_alt(&args,rec,int2nt(revint(ia)),int2nt(revint(ib)),0); }
- if ( ir==revint(ib) ) { args.nflip_swap++; return set_ref_alt(&args,rec,int2nt(revint(ib)),int2nt(revint(ia)),1); }
- error("FIXME: this should not happen %s:%d\n", bcf_seqname(args.hdr,rec),rec->pos+1);
- }
- else if ( args.mode==MODE_TOP2FWD )
- {
- int pair = 1 << ia | 1 << ib;
- if ( pair != 0x9 && pair != 0x6 ) // unambiguous pair: A/C or A/G
- {
- if ( ir==ia ) return ret;
-
- int ia_rev = revint(ia);
- if ( ir==ia_rev ) // vcfref is A, faref is T, flip
- {
- args.nflip++;
- return set_ref_alt(&args,rec,int2nt(ia_rev),int2nt(revint(ib)),0);
- }
- if ( ir==ib ) // vcfalt is faref, swap
- {
- args.nswap++;
- return set_ref_alt(&args,rec,int2nt(ib),int2nt(ia),1);
- }
- assert( ib==revint(ir) );
-
- args.nflip_swap++;
- return set_ref_alt(&args,rec,int2nt(revint(ib)),int2nt(revint(ia)),1);
- }
- else // ambiguous pair, sequence walking must be performed
- {
- int len, win = rec->pos > 100 ? 100 : rec->pos, beg = rec->pos - win, end = rec->pos + win;
- char *ref = faidx_fetch_seq(args.fai, (char*)bcf_seqname(args.hdr,rec), beg,end, &len);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", bcf_seqname(args.hdr,rec),rec->pos+1);
- if ( end - beg + 1 != len ) error("FIXME: check win=%d,len=%d at %s:%d (%d %d)\n", win,len, bcf_seqname(args.hdr,rec),rec->pos+1, end,beg);
-
- int i, mid = rec->pos - beg, strand = 0;
- for (i=1; i<=win; i++)
- {
- int ra = nt2int(ref[mid-i]);
- int rb = nt2int(ref[mid+i]);
- if ( ra<0 || rb<0 || ra==rb ) continue; // skip N's and non-infomative pairs: A/A, C/C, G/G, T/T
- pair = 1 << ra | 1 << rb;
- if ( pair==0x9 || pair==0x6 ) continue; // skip ambiguous pairs: A/T or C/G
- strand = 1 << ra & 0x9 ? 1 : -1;
- break;
- }
- free(ref);
-
- if ( strand==1 )
- {
- if ( ir==ia ) return ret;
- if ( ir==ib )
- {
- args.nswap++;
- return set_ref_alt(&args,rec,int2nt(ib),int2nt(ia),1);
- }
- }
- else if ( strand==-1 )
- {
- int ia_rev = revint(ia);
- int ib_rev = revint(ib);
- if ( ir==ia_rev )
- {
- args.nflip++;
- return set_ref_alt(&args,rec,int2nt(ia_rev),int2nt(ib_rev),0);
- }
- if ( ir==ib_rev )
- {
- args.nflip_swap++;
- return set_ref_alt(&args,rec,int2nt(ib_rev),int2nt(ia_rev),1);
- }
- }
-
- args.nunresolved++;
- return args.discard ? NULL : ret;
- }
- }
- return ret;
-}
-
-int top_mask[4][4] =
-{
- {0,1,1,1},
- {0,0,1,0},
- {0,0,0,0},
- {0,0,0,0},
-};
-int bot_mask[4][4] =
-{
- {0,0,0,0},
- {0,0,0,0},
- {0,1,0,0},
- {1,1,1,0},
-};
-
-void destroy(void)
-{
- uint32_t i,j,tot = 0;
- uint32_t top_err = 0, bot_err = 0;
- for (i=0; i<4; i++)
- {
- for (j=0; j<4; j++)
- {
- tot += args.count[i][j];
- if ( !top_mask[i][j] && args.count[i][j] ) top_err++;
- if ( !bot_mask[i][j] && args.count[i][j] ) bot_err++;
- }
- }
- uint32_t nskip = args.nonACGT+args.nonSNP+args.nonbiallelic;
- uint32_t ncmp = args.nsite - nskip;
-
- fprintf(bcftools_stderr,"# SC, guessed strand convention\n");
- fprintf(bcftools_stderr,"SC\tTOP-compatible\t%d\n",top_err?0:1);
- fprintf(bcftools_stderr,"SC\tBOT-compatible\t%d\n",bot_err?0:1);
-
- fprintf(bcftools_stderr,"# ST, substitution types\n");
- for (i=0; i<4; i++)
- {
- for (j=0; j<4; j++)
- {
- if ( i==j ) continue;
- fprintf(bcftools_stderr,"ST\t%c>%c\t%u\t%.1f%%\n", int2nt(i),int2nt(j),args.count[i][j], args.count[i][j]*100./tot);
- }
- }
- fprintf(bcftools_stderr,"# NS, Number of sites:\n");
- fprintf(bcftools_stderr,"NS\ttotal \t%u\n", args.nsite);
- fprintf(bcftools_stderr,"NS\tref match \t%u\t%.1f%%\n", args.nok,100.*args.nok/ncmp);
- fprintf(bcftools_stderr,"NS\tref mismatch \t%u\t%.1f%%\n", ncmp-args.nok,100.*(ncmp-args.nok)/ncmp);
- if ( args.mode!=MODE_STATS )
- {
- fprintf(bcftools_stderr,"NS\tflipped \t%u\t%.1f%%\n", args.nflip,100.*args.nflip/(args.nsite-nskip));
- fprintf(bcftools_stderr,"NS\tswapped \t%u\t%.1f%%\n", args.nswap,100.*args.nswap/(args.nsite-nskip));
- fprintf(bcftools_stderr,"NS\tflip+swap \t%u\t%.1f%%\n", args.nflip_swap,100.*args.nflip_swap/(args.nsite-nskip));
- fprintf(bcftools_stderr,"NS\tunresolved \t%u\t%.1f%%\n", args.nunresolved,100.*args.nunresolved/(args.nsite-nskip));
- fprintf(bcftools_stderr,"NS\tfixed pos \t%u\t%.1f%%\n", args.npos_err,100.*args.npos_err/(args.nsite-nskip));
- }
- fprintf(bcftools_stderr,"NS\tskipped \t%u\n", nskip);
- fprintf(bcftools_stderr,"NS\tnon-ACGT \t%u\n", args.nonACGT);
- fprintf(bcftools_stderr,"NS\tnon-SNP \t%u\n", args.nonSNP);
- fprintf(bcftools_stderr,"NS\tnon-biallelic\t%u\n", args.nonbiallelic);
-
- free(args.gts);
- if ( args.fai ) fai_destroy(args.fai);
- if ( args.i2m ) kh_destroy(i2m, args.i2m);
-}
+++ /dev/null
-/* plugins/frameshifts.c -- annotates frameshift indels.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <getopt.h>
-
-bcf_hdr_t *in_hdr, *out_hdr;
-bcf_sr_regions_t *exons;
-int32_t *frm = NULL, nfrm = 0;
-
-const char *about(void)
-{
- return
- "Annotate frameshift indels.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Annotate frameshift indels\n"
- "Usage: bcftools +frameshifts [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -e, --exons <file> list of exons, see \"--targets-file\" man page entry for details\n"
- "\n"
- "Example:\n"
- " bcftools +frameshifts in.vcf -- -e exons.bed.gz\n"
- "\n";
-}
-
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- int c;
- char *fname = NULL;
-
- static struct option loptions[] =
- {
- {"exons",1,0,'e'},
- {0,0,0,0}
- };
- while ((c = getopt_long(argc, argv, "e:?h",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'e': fname = optarg; break;
- case 'h':
- case '?':
- default: fprintf(stderr,"%s", usage()); exit(1); break;
- }
- }
- if ( !fname )
- {
- fprintf(stderr,"Missing the -e option.\n");
- return -1;
- }
-
- in_hdr = in;
- out_hdr = out;
-
- int ret = bcf_hdr_append(out_hdr,"##INFO=<ID=OOF,Number=A,Type=Integer,Description=\"Frameshift Indels: out-of-frame (1), in-frame (0), not-applicable (-1 or missing)\">");
- if ( ret!=0 )
- {
- fprintf(stderr,"Error updating the header\n");
- return -1;
- }
-
- exons = bcf_sr_regions_init(fname,1,0,1,2);
- if ( !exons )
- {
- fprintf(stderr,"Error occurred while reading (was the file compressed with bgzip?): %s\n", fname);
- return -1;
- }
-
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- if ( rec->n_allele<2 ) return rec; // not a variant
-
- int type = bcf_get_variant_types(rec);
- if ( !(type&VCF_INDEL) ) return rec; // not an indel
-
- int i, len = 0;
- for (i=1; i<rec->n_allele; i++)
- if ( len > rec->d.var[i].n ) len = rec->d.var[i].n;
-
- int pos_to = len!=0 ? rec->pos : rec->pos - len; // len is negative
- if ( bcf_sr_regions_overlap(exons, bcf_seqname(in_hdr,rec),rec->pos,pos_to) ) return rec; // no overlap
-
- hts_expand(int32_t,rec->n_allele-1,nfrm,frm);
- for (i=1; i<rec->n_allele; i++)
- {
- if ( rec->d.var[i].type!=VCF_INDEL ) { frm[i-1] = -1; continue; }
-
- int len = rec->d.var[i].n, tlen = 0;
- if ( len>0 )
- {
- // insertion
- if ( exons->start <= rec->pos && exons->end > rec->pos ) tlen = abs(len);
- }
- else if ( exons->start <= rec->pos + abs(len) )
- {
- // deletion
- tlen = abs(len);
- if ( rec->pos < exons->start ) // trim the beginning
- tlen -= exons->start - rec->pos + 1;
- if ( exons->end < rec->pos + abs(len) ) // trim the end
- tlen -= rec->pos + abs(len) - exons->end;
- }
- if ( tlen ) // there are some deleted/inserted bases in the exon
- {
- if ( tlen%3 ) frm[i-1] = 1; // out-of-frame
- else frm[i-1] = 0; // in-frame
- }
- else frm[i-1] = -1; // not applicable (is outside)
- }
-
- if ( bcf_update_info_int32(out_hdr,rec,"OOF",frm,rec->n_allele-1)<0 ) { fprintf(stderr, "Could not annotate OOF :-/\n"); exit(1); }
- return rec;
-}
-
-
-void destroy(void)
-{
- bcf_sr_regions_destroy(exons);
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugins/frameshifts.c -- annotates frameshift indels.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <getopt.h>
-
-bcf_hdr_t *in_hdr, *out_hdr;
-bcf_sr_regions_t *exons;
-int32_t *frm = NULL, nfrm = 0;
-
-const char *about(void)
-{
- return
- "Annotate frameshift indels.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Annotate frameshift indels\n"
- "Usage: bcftools +frameshifts [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -e, --exons <file> list of exons, see \"--targets-file\" man page entry for details\n"
- "\n"
- "Example:\n"
- " bcftools +frameshifts in.vcf -- -e exons.bed.gz\n"
- "\n";
-}
-
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- int c;
- char *fname = NULL;
-
- static struct option loptions[] =
- {
- {"exons",1,0,'e'},
- {0,0,0,0}
- };
- while ((c = getopt_long(argc, argv, "e:?h",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'e': fname = optarg; break;
- case 'h':
- case '?':
- default: fprintf(bcftools_stderr,"%s", usage()); exit(1); break;
- }
- }
- if ( !fname )
- {
- fprintf(bcftools_stderr,"Missing the -e option.\n");
- return -1;
- }
-
- in_hdr = in;
- out_hdr = out;
-
- int ret = bcf_hdr_append(out_hdr,"##INFO=<ID=OOF,Number=A,Type=Integer,Description=\"Frameshift Indels: out-of-frame (1), in-frame (0), not-applicable (-1 or missing)\">");
- if ( ret!=0 )
- {
- fprintf(bcftools_stderr,"Error updating the header\n");
- return -1;
- }
-
- exons = bcf_sr_regions_init(fname,1,0,1,2);
- if ( !exons )
- {
- fprintf(bcftools_stderr,"Error occurred while reading (was the file compressed with bgzip?): %s\n", fname);
- return -1;
- }
-
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- if ( rec->n_allele<2 ) return rec; // not a variant
-
- int type = bcf_get_variant_types(rec);
- if ( !(type&VCF_INDEL) ) return rec; // not an indel
-
- int i, len = 0;
- for (i=1; i<rec->n_allele; i++)
- if ( len > rec->d.var[i].n ) len = rec->d.var[i].n;
-
- int pos_to = len!=0 ? rec->pos : rec->pos - len; // len is negative
- if ( bcf_sr_regions_overlap(exons, bcf_seqname(in_hdr,rec),rec->pos,pos_to) ) return rec; // no overlap
-
- hts_expand(int32_t,rec->n_allele-1,nfrm,frm);
- for (i=1; i<rec->n_allele; i++)
- {
- if ( rec->d.var[i].type!=VCF_INDEL ) { frm[i-1] = -1; continue; }
-
- int len = rec->d.var[i].n, tlen = 0;
- if ( len>0 )
- {
- // insertion
- if ( exons->start <= rec->pos && exons->end > rec->pos ) tlen = abs(len);
- }
- else if ( exons->start <= rec->pos + abs(len) )
- {
- // deletion
- tlen = abs(len);
- if ( rec->pos < exons->start ) // trim the beginning
- tlen -= exons->start - rec->pos + 1;
- if ( exons->end < rec->pos + abs(len) ) // trim the end
- tlen -= rec->pos + abs(len) - exons->end;
- }
- if ( tlen ) // there are some deleted/inserted bases in the exon
- {
- if ( tlen%3 ) frm[i-1] = 1; // out-of-frame
- else frm[i-1] = 0; // in-frame
- }
- else frm[i-1] = -1; // not applicable (is outside)
- }
-
- if ( bcf_update_info_int32(out_hdr,rec,"OOF",frm,rec->n_allele-1)<0 ) { fprintf(bcftools_stderr, "Could not annotate OOF :-/\n"); exit(1); }
- return rec;
-}
-
-
-void destroy(void)
-{
- bcf_sr_regions_destroy(exons);
-}
-
-
+++ /dev/null
-/*
- Copyright (C) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <stdint.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include <inttypes.h>
-#include <unistd.h>
-#include "bcftools.h"
-#include "filter.h"
-
-// Logic of the filters: include or exclude sites which match the filters?
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-#define GUESS_GT 1
-#define GUESS_PL 2
-#define GUESS_GL 4
-
-typedef struct
-{
- uint64_t ncount;
- double phap, pdip;
-}
-count_t;
-
-typedef struct
-{
- char *chr;
- uint32_t start, end;
- count_t *counts; // per-sample counts: counts[isample]
-}
-stats_t;
-
-typedef struct
-{
- int argc;
- char **argv, *af_tag;
- double af_dflt;
- stats_t stats;
- filter_t *filter;
- char *filter_str;
- int filter_logic; // include or exclude sites which match the filters? One of FLT_INCLUDE/FLT_EXCLUDE
- const uint8_t *smpl_pass;
- int nsample, verbose, tag, include_indels;
- int *counts, ncounts; // number of observed GTs with given ploidy, used when -g is not given
- double *tmpf, *pl2p, gt_err_prob;
- float *af;
- int maf;
- int32_t *arr, narr, nfarr;
- float *farr;
- bcf_srs_t *sr;
- bcf_hdr_t *hdr;
-}
-args_t;
-
-const char *about(void)
-{
- return "Determine sample sex by checking genotype likelihoods in haploid regions.\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Determine sample sex by checking genotype likelihoods (GL,PL) or genotypes (GT)\n"
- " in the non-PAR region of chrX. The HWE is assumed, so given the alternate allele\n"
- " frequency fA and the genotype likelihoods pRR,pRA,pAA, the probabilities are\n"
- " calculated as\n"
- " P(dip) = pRR*(1-fA)^2 + pAA*fA^2 + 2*pRA*(1-fA)*fA\n"
- " P(hap) = pRR*(1-fA) + pAA*fA\n"
- " When genotype likelihoods are not available, the -e option is used to account\n"
- " for genotyping errors with -t GT. The alternate allele frequency fA is estimated\n"
- " directly from the data (the default) or can be provided by an INFO tag.\n"
- " The results can be visualized using the accompanied guess-ploidy.py script.\n"
- " Note that this plugin is intended to replace the former vcf2sex plugin.\n"
- "\n"
- "Usage: bcftools +guess-ploidy <file.vcf.gz> [Plugin Options]\n"
- "Plugin options:\n"
- " --AF-dflt <float> the default alternate allele frequency [0.5]\n"
- " --AF-tag <TAG> use TAG for allele frequency\n"
- " -e, --error-rate <float> probability of GT being wrong (with -t GT) [1e-3]\n"
- " --exclude <expr> exclude sites for which the expression is true\n"
- " -i, --include-indels do not skip indel sites\n"
- " --include <expr> include only sites for which the expression is true\n"
- " -g, --genome <str> shortcut to select nonPAR region for common genomes b37|hg19|b38|hg38\n"
- " -r, --regions <chr:beg-end> restrict to comma-separated list of regions\n"
- " -R, --regions-file <file> restrict to regions listed in a file\n"
- " -t, --tag <tag> genotype or genotype likelihoods: GT, PL, GL [PL]\n"
- " -v, --verbose verbose output (specify twice to increase verbosity)\n"
- "\n"
- "Region shortcuts:\n"
- " b37 .. -r X:2699521-154931043 # GRCh37 no-chr prefix\n"
- " b38 .. -r X:2781480-155701381 # GRCh38 no-chr prefix\n"
- " hg19 .. -r chrX:2699521-154931043 # GRCh37 chr prefix\n"
- " hg38 .. -r chrX:2781480-155701381 # GRCh38 chr prefix\n"
- "\n"
- "Examples:\n"
- " bcftools +guess-ploidy -g b37 in.vcf.gz\n"
- " bcftools +guess-ploidy in.vcf.gz -t GL -r chrX:2699521-154931043\n"
- " bcftools view file.vcf.gz -r chrX:2699521-154931043 | bcftools +guess-ploidy\n"
- " bcftools +guess-ploidy in.bcf -v > ploidy.txt && guess-ploidy.py ploidy.txt img\n"
- "\n";
-}
-
-static inline int smpl_pass(args_t *args, int ismpl)
-{
- if ( !args->smpl_pass ) return 1;
- int pass = args->smpl_pass[ismpl];
- if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
- if ( pass ) return 1;
- return 0;
-}
-
-void process_region_guess(args_t *args)
-{
- while ( bcf_sr_next_line(args->sr) )
- {
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- if ( rec->n_allele==1 ) continue;
- if ( !args->include_indels && !(bcf_get_variant_types(rec)&VCF_SNP) ) continue;
-
- if ( args->filter )
- {
- int pass = filter_test(args->filter, rec, &args->smpl_pass);
- if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
- if ( !args->smpl_pass && !pass ) continue; // site-level filtering, not per-sample filtering
- }
-
- double freq[2] = {0,0}, sum;
- int ismpl,i;
- if ( args->tag & GUESS_GT ) // use GTs to guess the ploidy, considering only one ALT
- {
- int ngt = bcf_get_genotypes(args->hdr,rec,&args->arr,&args->narr);
- if ( ngt<=0 ) continue;
- ngt /= args->nsample;
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- int32_t *ptr = args->arr + ismpl*ngt;
- double *tmp = args->tmpf + ismpl*3;
-
- if ( ptr[0]==bcf_gt_missing )
- {
- tmp[0] = -1;
- continue;
- }
- if ( ptr[1]==bcf_int32_vector_end )
- {
- if ( bcf_gt_allele(ptr[0])==0 ) // haploid R
- {
- tmp[0] = 1 - 2*args->gt_err_prob;
- tmp[1] = tmp[2] = args->gt_err_prob;
- }
- else // haploid A
- {
- tmp[0] = tmp[1] = args->gt_err_prob;
- tmp[2] = 1 - 2*args->gt_err_prob;
- }
- continue;
- }
- if ( bcf_gt_allele(ptr[0])==0 && bcf_gt_allele(ptr[1])==0 ) // RR
- {
- tmp[0] = 1 - 2*args->gt_err_prob;
- tmp[1] = tmp[2] = args->gt_err_prob;
- }
- else if ( bcf_gt_allele(ptr[0])==bcf_gt_allele(ptr[1]) ) // AA
- {
- tmp[0] = tmp[1] = args->gt_err_prob;
- tmp[2] = 1 - 2*args->gt_err_prob;
- }
- else // RA or hetAA, treating as RA
- {
- tmp[1] = 1 - 2*args->gt_err_prob;
- tmp[0] = tmp[2] = args->gt_err_prob;
- }
- freq[0] += 2*tmp[0]+tmp[1];
- freq[1] += tmp[1]+2*tmp[2];
- }
- }
- else if ( args->tag & GUESS_PL ) // use PL guess the ploidy, restrict to first ALT allele
- {
- int npl = bcf_get_format_int32(args->hdr,rec,"PL",&args->arr,&args->narr);
- if ( npl<=0 ) continue;
- npl /= args->nsample;
- int ndip_gt = rec->n_allele*(rec->n_allele+1)/2;
- if ( npl==ndip_gt ) // diploid
- {
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- int32_t *ptr = args->arr + ismpl*npl;
- double *tmp = args->tmpf + ismpl*3;
-
- // restrict to first ALT
- if ( ptr[0]==bcf_int32_missing || ptr[1]==bcf_int32_missing || ptr[2]==bcf_int32_missing )
- {
- tmp[0] = -1;
- continue;
- }
- if ( ptr[0]==ptr[1] && ptr[0]==ptr[2] ) // non-informative
- {
- tmp[0] = -1;
- continue;
- }
- if ( ptr[2]==bcf_int32_vector_end )
- {
- tmp[0] = (ptr[0]<0 || ptr[0]>=256) ? args->pl2p[255] : args->pl2p[ptr[0]];
- tmp[1] = args->pl2p[255];
- tmp[2] = (ptr[1]<0 || ptr[1]>=256) ? args->pl2p[255] : args->pl2p[ptr[1]];
- }
- else
- for (i=0; i<3; i++)
- tmp[i] = (ptr[i]<0 || ptr[i]>=256) ? args->pl2p[255] : args->pl2p[ptr[i]];
-
- sum = 0;
- for (i=0; i<3; i++) sum += tmp[i];
- for (i=0; i<3; i++) tmp[i] /= sum;
-
- if ( ptr[2]==bcf_int32_vector_end )
- {
- freq[0] += tmp[0];
- freq[1] += tmp[2];
- }
- else
- {
- freq[0] += 2*tmp[0]+tmp[1];
- freq[1] += tmp[1]+2*tmp[2];
- }
- }
- }
- else if ( npl==rec->n_allele ) // all samples haploid
- {
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- int32_t *ptr = args->arr + ismpl*npl;
- double *tmp = args->tmpf + ismpl*3;
-
- // restrict to first ALT
- if ( ptr[0]==bcf_int32_missing || ptr[1]==bcf_int32_missing )
- {
- tmp[0] = -1;
- continue;
- }
- tmp[0] = (ptr[0]<0 || ptr[0]>=256) ? args->pl2p[255] : args->pl2p[ptr[0]];
- tmp[1] = args->pl2p[255];
- tmp[2] = (ptr[1]<0 || ptr[1]>=256) ? args->pl2p[255] : args->pl2p[ptr[1]];
-
- sum = 0;
- for (i=0; i<3; i++) sum += tmp[i];
- for (i=0; i<3; i++) tmp[i] /= sum;
-
- freq[0] += tmp[0];
- freq[1] += tmp[2];
- }
- }
- else
- continue; // neither diploid nor haploid
- }
- else // use GL
- {
- int ngl = bcf_get_format_float(args->hdr,rec,"GL",&args->farr,&args->nfarr);
- if ( ngl<=0 ) continue;
- ngl /= args->nsample;
- int ndip_gt = rec->n_allele*(rec->n_allele+1)/2;
- if ( ngl==ndip_gt ) // diploid
- {
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- float *ptr = args->farr + ismpl*ngl;
- double *tmp = args->tmpf + ismpl*3;
-
- // restrict to first ALT
- if ( bcf_float_is_missing(ptr[0]) || bcf_float_is_missing(ptr[1]) || bcf_float_is_missing(ptr[2]) )
- {
- tmp[0] = -1;
- continue;
- }
- if ( ptr[0]==ptr[1] && ptr[0]==ptr[2] ) // non-informative
- {
- tmp[0] = -1;
- continue;
- }
- if ( bcf_float_is_vector_end(ptr[2]) )
- {
- tmp[0] = pow(10.,ptr[0]);
- tmp[1] = 1e-26; // arbitrary small value for a het
- tmp[2] = pow(10.,ptr[1]);
- }
- else
- for (i=0; i<3; i++)
- tmp[i] = pow(10.,ptr[i]);
-
- sum = 0;
- for (i=0; i<3; i++) sum += tmp[i];
- for (i=0; i<3; i++) tmp[i] /= sum;
-
- if ( bcf_float_is_vector_end(ptr[2]) )
- {
- freq[0] += tmp[0];
- freq[1] += tmp[2];
- }
- else
- {
- freq[0] += 2*tmp[0]+tmp[1];
- freq[1] += tmp[1]+2*tmp[2];
- }
- }
- }
- else if ( ngl==rec->n_allele ) // all samples haploid
- {
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- float *ptr = args->farr + ismpl*ngl;
- double *tmp = args->tmpf + ismpl*3;
-
- // restrict to first ALT
- if ( bcf_float_is_missing(ptr[0]) || bcf_float_is_missing(ptr[1]) )
- {
- tmp[0] = -1;
- continue;
- }
- tmp[0] = pow(10.,ptr[0]);
- tmp[1] = 1e-26;
- tmp[2] = pow(10.,ptr[1]);
-
- sum = 0;
- for (i=0; i<3; i++) sum += tmp[i];
- for (i=0; i<3; i++) tmp[i] /= sum;
-
- freq[0] += tmp[0];
- freq[1] += tmp[2];
- }
- }
- else
- continue; // neither diploid nor haploid
- }
- if ( args->af_tag )
- {
- int ret = bcf_get_info_float(args->hdr,rec,args->af_tag,&args->af, &args->maf);
- if ( ret>0 ) { freq[0] = 1 - args->af[0]; freq[1] = args->af[0]; }
- }
-
- if ( !freq[0] && !freq[1] ) { freq[0] = 1 - args->af_dflt; freq[1] = args->af_dflt; }
- sum = freq[0] + freq[1];
- freq[0] /= sum;
- freq[1] /= sum;
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- count_t *counts = &args->stats.counts[ismpl];
- double *tmp = args->tmpf + ismpl*3;
- if ( tmp[0] < 0 ) continue;
- double phap = freq[0]*tmp[0] + freq[1]*tmp[2];
- double pdip = freq[0]*freq[0]*tmp[0] + 2*freq[0]*freq[1]*tmp[1] + freq[1]*freq[1]*tmp[2];
- counts->phap += log(phap);
- counts->pdip += log(pdip);
- counts->ncount++;
- if ( args->verbose>1 )
- printf("DBG\t%s\t%d\t%s\t%e\t%e\t%e\t%e\t%e\t%e\n", bcf_seqname(args->hdr,rec),rec->pos+1,bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,ismpl),
- freq[1],tmp[0],tmp[1],tmp[2],phap,pdip);
- }
- }
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->tag = GUESS_PL;
- args->argc = argc; args->argv = argv;
- args->gt_err_prob = 1e-3;
- args->af_dflt = 0.5;
- char *region = NULL;
- int region_is_file = 0;
- static struct option loptions[] =
- {
- {"AF-tag",required_argument,NULL,0},
- {"AF-dflt",required_argument,NULL,1},
- {"exclude",required_argument,NULL,2},
- {"include",required_argument,NULL,3},
- {"verbose",no_argument,NULL,'v'},
- {"include-indels",no_argument,NULL,'i'},
- {"error-rate",required_argument,NULL,'e'},
- {"tag",required_argument,NULL,'t'},
- {"genome",required_argument,NULL,'g'},
- {"regions",required_argument,NULL,'r'},
- {"regions-file",required_argument,NULL,'R'},
- {"background",required_argument,NULL,'b'},
- {NULL,0,NULL,0}
- };
- int c;
- char *tmp;
- while ((c = getopt_long(argc, argv, "vr:R:t:e:ig:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 0: args->af_tag = optarg; break;
- case 1:
- args->af_dflt = strtod(optarg,&tmp);
- if ( *tmp ) error("Could not parse: --AF-dflt %s\n", optarg);
- break;
- case 2: args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 3: args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 'i': args->include_indels = 1; break;
- case 'e':
- args->gt_err_prob = strtod(optarg,&tmp);
- if ( *tmp ) error("Could not parse: -e %s\n", optarg);
- if ( args->gt_err_prob<0 || args->gt_err_prob>1 ) error("Expected value from the interval [0,1]: -e %s\n", optarg);
- break;
- case 'g':
- if ( !strcasecmp(optarg,"b37") ) region = "X:2699521-154931043";
- else if ( !strcasecmp(optarg,"b38") ) region = "X:2781480-155701381";
- else if ( !strcasecmp(optarg,"hg19") ) region = "chrX:2699521-154931043";
- else if ( !strcasecmp(optarg,"hg38") ) region = "chrX:2781480-155701381";
- else error("The argument not recognised, expected --genome b37, b38, hg19 or hg38: %s\n", optarg);
- break;
- case 'R': region_is_file = 1;
- case 'r': region = optarg; break;
- case 'v': args->verbose++; break;
- case 't':
- if ( !strcasecmp(optarg,"GT") ) args->tag = GUESS_GT;
- else if ( !strcasecmp(optarg,"PL") ) args->tag = GUESS_PL;
- else if ( !strcasecmp(optarg,"GL") ) args->tag = GUESS_GL;
- else error("The argument not recognised, expected --tag GT, PL or GL: %s\n", optarg);
- break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( args->filter_logic == (FLT_EXCLUDE|FLT_INCLUDE) ) error("Only one of --include or --exclude can be given.\n");
-
- char *fname = NULL;
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) fname = "-"; // reading from stdin
- else { error("%s",usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s",usage_text());
- else fname = argv[optind];
-
- args->sr = bcf_sr_init();
- if ( strcmp("-",fname) )
- {
- if ( region )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, region, region_is_file)<0 )
- error("Failed to read the regions: %s\n",region);
- }
- }
- else
- {
- if ( region )
- {
- if ( bcf_sr_set_targets(args->sr, region, region_is_file, 0)<0 )
- error("Failed to read the targets: %s\n",region);
- }
- }
- if ( !bcf_sr_add_reader(args->sr,fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr = args->sr->readers[0].header;
- args->nsample = bcf_hdr_nsamples(args->hdr);
- args->stats.counts = (count_t*) calloc(args->nsample,sizeof(count_t));
-
- if ( args->filter_str )
- args->filter = filter_init(args->hdr, args->filter_str);
-
- if ( args->af_tag && !bcf_hdr_idinfo_exists(args->hdr,BCF_HL_INFO,bcf_hdr_id2int(args->hdr,BCF_DT_ID,args->af_tag)) )
- error("No such INFO tag: %s\n", args->af_tag);
-
- if ( args->tag&GUESS_PL && bcf_hdr_id2int(args->hdr, BCF_DT_ID, "PL")<0 )
- {
- fprintf(stderr, "Warning: PL tag not found in header, switching to GL\n");
- args->tag = GUESS_GL;
- }
-
- if ( args->tag&GUESS_GL && bcf_hdr_id2int(args->hdr, BCF_DT_ID, "GL")<0 )
- {
- fprintf(stderr, "Warning: GL tag not found in header, switching to GT\n");
- args->tag = GUESS_GT;
- }
-
- if ( args->tag&GUESS_GT && bcf_hdr_id2int(args->hdr, BCF_DT_ID, "GT")<0 )
- error("Error: GT tag not found in header\n");
-
- int i;
- if ( args->tag&GUESS_PL )
- {
- args->pl2p = (double*) calloc(256,sizeof(double));
- for (i=0; i<256; i++) args->pl2p[i] = pow(10., -i/10.);
- }
- if ( args->tag&GUESS_PL || args->tag&GUESS_GL || args->tag&GUESS_GT )
- args->tmpf = (double*) malloc(sizeof(*args->tmpf)*3*args->nsample);
-
- if ( args->verbose )
- {
- printf("# This file was produced by: bcftools +guess-ploidy(%s+htslib-%s)\n", bcftools_version(),hts_version());
- printf("# The command line was:\tbcftools +%s", args->argv[0]);
- for (i=1; i<args->argc; i++)
- printf(" %s",args->argv[i]);
- printf("\n");
- printf("# [1]SEX\t[2]Sample\t[3]Predicted sex\t[4]log P(Haploid)/nSites\t[5]log P(Diploid)/nSites\t[6]nSites\t[7]Score: F < 0 < M ($4-$5)\n");
- if ( args->verbose>1 )
- printf("# [1]DBG\t[2]Chr\t[3]Pos\t[4]Sample\t[5]AF\t[6]pRR\t[7]pRA\t[8]pAA\t[9]P(Haploid)\t[10]P(Diploid)\n");
- }
-
- process_region_guess(args);
-
- for (i=0; i<args->nsample; i++)
- {
- double phap = args->stats.counts[i].ncount ? args->stats.counts[i].phap / args->stats.counts[i].ncount : 0.5;
- double pdip = args->stats.counts[i].ncount ? args->stats.counts[i].pdip / args->stats.counts[i].ncount : 0.5;
- char predicted_sex = 'U';
- if (phap>pdip) predicted_sex = 'M';
- else if (phap<pdip) predicted_sex = 'F';
- if ( args->verbose )
- {
- printf("SEX\t%s\t%c\t%f\t%f\t%"PRId64"\t%f\n", args->hdr->samples[i],predicted_sex,
- phap,pdip,args->stats.counts[i].ncount,phap-pdip);
- }
- else
- printf("%s\t%c\n", args->hdr->samples[i],predicted_sex);
- }
-
- if ( args->filter )
- filter_destroy(args->filter);
-
- bcf_sr_destroy(args->sr);
- free(args->pl2p);
- free(args->tmpf);
- free(args->counts);
- free(args->stats.counts);
- free(args->arr);
- free(args->farr);
- free(args->af);
- free(args);
- return 0;
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/*
- Copyright (C) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <stdint.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include <inttypes.h>
-#include <unistd.h>
-#include "bcftools.h"
-#include "filter.h"
-
-// Logic of the filters: include or exclude sites which match the filters?
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-#define GUESS_GT 1
-#define GUESS_PL 2
-#define GUESS_GL 4
-
-typedef struct
-{
- uint64_t ncount;
- double phap, pdip;
-}
-count_t;
-
-typedef struct
-{
- char *chr;
- uint32_t start, end;
- count_t *counts; // per-sample counts: counts[isample]
-}
-stats_t;
-
-typedef struct
-{
- int argc;
- char **argv, *af_tag;
- double af_dflt;
- stats_t stats;
- filter_t *filter;
- char *filter_str;
- int filter_logic; // include or exclude sites which match the filters? One of FLT_INCLUDE/FLT_EXCLUDE
- const uint8_t *smpl_pass;
- int nsample, verbose, tag, include_indels;
- int *counts, ncounts; // number of observed GTs with given ploidy, used when -g is not given
- double *tmpf, *pl2p, gt_err_prob;
- float *af;
- int maf;
- int32_t *arr, narr, nfarr;
- float *farr;
- bcf_srs_t *sr;
- bcf_hdr_t *hdr;
-}
-args_t;
-
-const char *about(void)
-{
- return "Determine sample sex by checking genotype likelihoods in haploid regions.\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Determine sample sex by checking genotype likelihoods (GL,PL) or genotypes (GT)\n"
- " in the non-PAR region of chrX. The HWE is assumed, so given the alternate allele\n"
- " frequency fA and the genotype likelihoods pRR,pRA,pAA, the probabilities are\n"
- " calculated as\n"
- " P(dip) = pRR*(1-fA)^2 + pAA*fA^2 + 2*pRA*(1-fA)*fA\n"
- " P(hap) = pRR*(1-fA) + pAA*fA\n"
- " When genotype likelihoods are not available, the -e option is used to account\n"
- " for genotyping errors with -t GT. The alternate allele frequency fA is estimated\n"
- " directly from the data (the default) or can be provided by an INFO tag.\n"
- " The results can be visualized using the accompanied guess-ploidy.py script.\n"
- " Note that this plugin is intended to replace the former vcf2sex plugin.\n"
- "\n"
- "Usage: bcftools +guess-ploidy <file.vcf.gz> [Plugin Options]\n"
- "Plugin options:\n"
- " --AF-dflt <float> the default alternate allele frequency [0.5]\n"
- " --AF-tag <TAG> use TAG for allele frequency\n"
- " -e, --error-rate <float> probability of GT being wrong (with -t GT) [1e-3]\n"
- " --exclude <expr> exclude sites for which the expression is true\n"
- " -i, --include-indels do not skip indel sites\n"
- " --include <expr> include only sites for which the expression is true\n"
- " -g, --genome <str> shortcut to select nonPAR region for common genomes b37|hg19|b38|hg38\n"
- " -r, --regions <chr:beg-end> restrict to comma-separated list of regions\n"
- " -R, --regions-file <file> restrict to regions listed in a file\n"
- " -t, --tag <tag> genotype or genotype likelihoods: GT, PL, GL [PL]\n"
- " -v, --verbose verbose output (specify twice to increase verbosity)\n"
- "\n"
- "Region shortcuts:\n"
- " b37 .. -r X:2699521-154931043 # GRCh37 no-chr prefix\n"
- " b38 .. -r X:2781480-155701381 # GRCh38 no-chr prefix\n"
- " hg19 .. -r chrX:2699521-154931043 # GRCh37 chr prefix\n"
- " hg38 .. -r chrX:2781480-155701381 # GRCh38 chr prefix\n"
- "\n"
- "Examples:\n"
- " bcftools +guess-ploidy -g b37 in.vcf.gz\n"
- " bcftools +guess-ploidy in.vcf.gz -t GL -r chrX:2699521-154931043\n"
- " bcftools view file.vcf.gz -r chrX:2699521-154931043 | bcftools +guess-ploidy\n"
- " bcftools +guess-ploidy in.bcf -v > ploidy.txt && guess-ploidy.py ploidy.txt img\n"
- "\n";
-}
-
-static inline int smpl_pass(args_t *args, int ismpl)
-{
- if ( !args->smpl_pass ) return 1;
- int pass = args->smpl_pass[ismpl];
- if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
- if ( pass ) return 1;
- return 0;
-}
-
-void process_region_guess(args_t *args)
-{
- while ( bcf_sr_next_line(args->sr) )
- {
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- if ( rec->n_allele==1 ) continue;
- if ( !args->include_indels && !(bcf_get_variant_types(rec)&VCF_SNP) ) continue;
-
- if ( args->filter )
- {
- int pass = filter_test(args->filter, rec, &args->smpl_pass);
- if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
- if ( !args->smpl_pass && !pass ) continue; // site-level filtering, not per-sample filtering
- }
-
- double freq[2] = {0,0}, sum;
- int ismpl,i;
- if ( args->tag & GUESS_GT ) // use GTs to guess the ploidy, considering only one ALT
- {
- int ngt = bcf_get_genotypes(args->hdr,rec,&args->arr,&args->narr);
- if ( ngt<=0 ) continue;
- ngt /= args->nsample;
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- int32_t *ptr = args->arr + ismpl*ngt;
- double *tmp = args->tmpf + ismpl*3;
-
- if ( ptr[0]==bcf_gt_missing )
- {
- tmp[0] = -1;
- continue;
- }
- if ( ptr[1]==bcf_int32_vector_end )
- {
- if ( bcf_gt_allele(ptr[0])==0 ) // haploid R
- {
- tmp[0] = 1 - 2*args->gt_err_prob;
- tmp[1] = tmp[2] = args->gt_err_prob;
- }
- else // haploid A
- {
- tmp[0] = tmp[1] = args->gt_err_prob;
- tmp[2] = 1 - 2*args->gt_err_prob;
- }
- continue;
- }
- if ( bcf_gt_allele(ptr[0])==0 && bcf_gt_allele(ptr[1])==0 ) // RR
- {
- tmp[0] = 1 - 2*args->gt_err_prob;
- tmp[1] = tmp[2] = args->gt_err_prob;
- }
- else if ( bcf_gt_allele(ptr[0])==bcf_gt_allele(ptr[1]) ) // AA
- {
- tmp[0] = tmp[1] = args->gt_err_prob;
- tmp[2] = 1 - 2*args->gt_err_prob;
- }
- else // RA or hetAA, treating as RA
- {
- tmp[1] = 1 - 2*args->gt_err_prob;
- tmp[0] = tmp[2] = args->gt_err_prob;
- }
- freq[0] += 2*tmp[0]+tmp[1];
- freq[1] += tmp[1]+2*tmp[2];
- }
- }
- else if ( args->tag & GUESS_PL ) // use PL guess the ploidy, restrict to first ALT allele
- {
- int npl = bcf_get_format_int32(args->hdr,rec,"PL",&args->arr,&args->narr);
- if ( npl<=0 ) continue;
- npl /= args->nsample;
- int ndip_gt = rec->n_allele*(rec->n_allele+1)/2;
- if ( npl==ndip_gt ) // diploid
- {
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- int32_t *ptr = args->arr + ismpl*npl;
- double *tmp = args->tmpf + ismpl*3;
-
- // restrict to first ALT
- if ( ptr[0]==bcf_int32_missing || ptr[1]==bcf_int32_missing || ptr[2]==bcf_int32_missing )
- {
- tmp[0] = -1;
- continue;
- }
- if ( ptr[0]==ptr[1] && ptr[0]==ptr[2] ) // non-informative
- {
- tmp[0] = -1;
- continue;
- }
- if ( ptr[2]==bcf_int32_vector_end )
- {
- tmp[0] = (ptr[0]<0 || ptr[0]>=256) ? args->pl2p[255] : args->pl2p[ptr[0]];
- tmp[1] = args->pl2p[255];
- tmp[2] = (ptr[1]<0 || ptr[1]>=256) ? args->pl2p[255] : args->pl2p[ptr[1]];
- }
- else
- for (i=0; i<3; i++)
- tmp[i] = (ptr[i]<0 || ptr[i]>=256) ? args->pl2p[255] : args->pl2p[ptr[i]];
-
- sum = 0;
- for (i=0; i<3; i++) sum += tmp[i];
- for (i=0; i<3; i++) tmp[i] /= sum;
-
- if ( ptr[2]==bcf_int32_vector_end )
- {
- freq[0] += tmp[0];
- freq[1] += tmp[2];
- }
- else
- {
- freq[0] += 2*tmp[0]+tmp[1];
- freq[1] += tmp[1]+2*tmp[2];
- }
- }
- }
- else if ( npl==rec->n_allele ) // all samples haploid
- {
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- int32_t *ptr = args->arr + ismpl*npl;
- double *tmp = args->tmpf + ismpl*3;
-
- // restrict to first ALT
- if ( ptr[0]==bcf_int32_missing || ptr[1]==bcf_int32_missing )
- {
- tmp[0] = -1;
- continue;
- }
- tmp[0] = (ptr[0]<0 || ptr[0]>=256) ? args->pl2p[255] : args->pl2p[ptr[0]];
- tmp[1] = args->pl2p[255];
- tmp[2] = (ptr[1]<0 || ptr[1]>=256) ? args->pl2p[255] : args->pl2p[ptr[1]];
-
- sum = 0;
- for (i=0; i<3; i++) sum += tmp[i];
- for (i=0; i<3; i++) tmp[i] /= sum;
-
- freq[0] += tmp[0];
- freq[1] += tmp[2];
- }
- }
- else
- continue; // neither diploid nor haploid
- }
- else // use GL
- {
- int ngl = bcf_get_format_float(args->hdr,rec,"GL",&args->farr,&args->nfarr);
- if ( ngl<=0 ) continue;
- ngl /= args->nsample;
- int ndip_gt = rec->n_allele*(rec->n_allele+1)/2;
- if ( ngl==ndip_gt ) // diploid
- {
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- float *ptr = args->farr + ismpl*ngl;
- double *tmp = args->tmpf + ismpl*3;
-
- // restrict to first ALT
- if ( bcf_float_is_missing(ptr[0]) || bcf_float_is_missing(ptr[1]) || bcf_float_is_missing(ptr[2]) )
- {
- tmp[0] = -1;
- continue;
- }
- if ( ptr[0]==ptr[1] && ptr[0]==ptr[2] ) // non-informative
- {
- tmp[0] = -1;
- continue;
- }
- if ( bcf_float_is_vector_end(ptr[2]) )
- {
- tmp[0] = pow(10.,ptr[0]);
- tmp[1] = 1e-26; // arbitrary small value for a het
- tmp[2] = pow(10.,ptr[1]);
- }
- else
- for (i=0; i<3; i++)
- tmp[i] = pow(10.,ptr[i]);
-
- sum = 0;
- for (i=0; i<3; i++) sum += tmp[i];
- for (i=0; i<3; i++) tmp[i] /= sum;
-
- if ( bcf_float_is_vector_end(ptr[2]) )
- {
- freq[0] += tmp[0];
- freq[1] += tmp[2];
- }
- else
- {
- freq[0] += 2*tmp[0]+tmp[1];
- freq[1] += tmp[1]+2*tmp[2];
- }
- }
- }
- else if ( ngl==rec->n_allele ) // all samples haploid
- {
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- float *ptr = args->farr + ismpl*ngl;
- double *tmp = args->tmpf + ismpl*3;
-
- // restrict to first ALT
- if ( bcf_float_is_missing(ptr[0]) || bcf_float_is_missing(ptr[1]) )
- {
- tmp[0] = -1;
- continue;
- }
- tmp[0] = pow(10.,ptr[0]);
- tmp[1] = 1e-26;
- tmp[2] = pow(10.,ptr[1]);
-
- sum = 0;
- for (i=0; i<3; i++) sum += tmp[i];
- for (i=0; i<3; i++) tmp[i] /= sum;
-
- freq[0] += tmp[0];
- freq[1] += tmp[2];
- }
- }
- else
- continue; // neither diploid nor haploid
- }
- if ( args->af_tag )
- {
- int ret = bcf_get_info_float(args->hdr,rec,args->af_tag,&args->af, &args->maf);
- if ( ret>0 ) { freq[0] = 1 - args->af[0]; freq[1] = args->af[0]; }
- }
-
- if ( !freq[0] && !freq[1] ) { freq[0] = 1 - args->af_dflt; freq[1] = args->af_dflt; }
- sum = freq[0] + freq[1];
- freq[0] /= sum;
- freq[1] /= sum;
- for (ismpl=0; ismpl<args->nsample; ismpl++)
- {
- if ( !smpl_pass(args,ismpl) ) continue;
- count_t *counts = &args->stats.counts[ismpl];
- double *tmp = args->tmpf + ismpl*3;
- if ( tmp[0] < 0 ) continue;
- double phap = freq[0]*tmp[0] + freq[1]*tmp[2];
- double pdip = freq[0]*freq[0]*tmp[0] + 2*freq[0]*freq[1]*tmp[1] + freq[1]*freq[1]*tmp[2];
- counts->phap += log(phap);
- counts->pdip += log(pdip);
- counts->ncount++;
- if ( args->verbose>1 )
- fprintf(bcftools_stdout, "DBG\t%s\t%d\t%s\t%e\t%e\t%e\t%e\t%e\t%e\n", bcf_seqname(args->hdr,rec),rec->pos+1,bcf_hdr_int2id(args->hdr,BCF_DT_SAMPLE,ismpl),
- freq[1],tmp[0],tmp[1],tmp[2],phap,pdip);
- }
- }
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->tag = GUESS_PL;
- args->argc = argc; args->argv = argv;
- args->gt_err_prob = 1e-3;
- args->af_dflt = 0.5;
- char *region = NULL;
- int region_is_file = 0;
- static struct option loptions[] =
- {
- {"AF-tag",required_argument,NULL,0},
- {"AF-dflt",required_argument,NULL,1},
- {"exclude",required_argument,NULL,2},
- {"include",required_argument,NULL,3},
- {"verbose",no_argument,NULL,'v'},
- {"include-indels",no_argument,NULL,'i'},
- {"error-rate",required_argument,NULL,'e'},
- {"tag",required_argument,NULL,'t'},
- {"genome",required_argument,NULL,'g'},
- {"regions",required_argument,NULL,'r'},
- {"regions-file",required_argument,NULL,'R'},
- {"background",required_argument,NULL,'b'},
- {NULL,0,NULL,0}
- };
- int c;
- char *tmp;
- while ((c = getopt_long(argc, argv, "vr:R:t:e:ig:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 0: args->af_tag = optarg; break;
- case 1:
- args->af_dflt = strtod(optarg,&tmp);
- if ( *tmp ) error("Could not parse: --AF-dflt %s\n", optarg);
- break;
- case 2: args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 3: args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 'i': args->include_indels = 1; break;
- case 'e':
- args->gt_err_prob = strtod(optarg,&tmp);
- if ( *tmp ) error("Could not parse: -e %s\n", optarg);
- if ( args->gt_err_prob<0 || args->gt_err_prob>1 ) error("Expected value from the interval [0,1]: -e %s\n", optarg);
- break;
- case 'g':
- if ( !strcasecmp(optarg,"b37") ) region = "X:2699521-154931043";
- else if ( !strcasecmp(optarg,"b38") ) region = "X:2781480-155701381";
- else if ( !strcasecmp(optarg,"hg19") ) region = "chrX:2699521-154931043";
- else if ( !strcasecmp(optarg,"hg38") ) region = "chrX:2781480-155701381";
- else error("The argument not recognised, expected --genome b37, b38, hg19 or hg38: %s\n", optarg);
- break;
- case 'R': region_is_file = 1;
- case 'r': region = optarg; break;
- case 'v': args->verbose++; break;
- case 't':
- if ( !strcasecmp(optarg,"GT") ) args->tag = GUESS_GT;
- else if ( !strcasecmp(optarg,"PL") ) args->tag = GUESS_PL;
- else if ( !strcasecmp(optarg,"GL") ) args->tag = GUESS_GL;
- else error("The argument not recognised, expected --tag GT, PL or GL: %s\n", optarg);
- break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( args->filter_logic == (FLT_EXCLUDE|FLT_INCLUDE) ) error("Only one of --include or --exclude can be given.\n");
-
- char *fname = NULL;
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) fname = "-"; // reading from stdin
- else { error("%s",usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s",usage_text());
- else fname = argv[optind];
-
- args->sr = bcf_sr_init();
- if ( strcmp("-",fname) )
- {
- if ( region )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, region, region_is_file)<0 )
- error("Failed to read the regions: %s\n",region);
- }
- }
- else
- {
- if ( region )
- {
- if ( bcf_sr_set_targets(args->sr, region, region_is_file, 0)<0 )
- error("Failed to read the targets: %s\n",region);
- }
- }
- if ( !bcf_sr_add_reader(args->sr,fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr = args->sr->readers[0].header;
- args->nsample = bcf_hdr_nsamples(args->hdr);
- args->stats.counts = (count_t*) calloc(args->nsample,sizeof(count_t));
-
- if ( args->filter_str )
- args->filter = filter_init(args->hdr, args->filter_str);
-
- if ( args->af_tag && !bcf_hdr_idinfo_exists(args->hdr,BCF_HL_INFO,bcf_hdr_id2int(args->hdr,BCF_DT_ID,args->af_tag)) )
- error("No such INFO tag: %s\n", args->af_tag);
-
- if ( args->tag&GUESS_PL && bcf_hdr_id2int(args->hdr, BCF_DT_ID, "PL")<0 )
- {
- fprintf(bcftools_stderr, "Warning: PL tag not found in header, switching to GL\n");
- args->tag = GUESS_GL;
- }
-
- if ( args->tag&GUESS_GL && bcf_hdr_id2int(args->hdr, BCF_DT_ID, "GL")<0 )
- {
- fprintf(bcftools_stderr, "Warning: GL tag not found in header, switching to GT\n");
- args->tag = GUESS_GT;
- }
-
- if ( args->tag&GUESS_GT && bcf_hdr_id2int(args->hdr, BCF_DT_ID, "GT")<0 )
- error("Error: GT tag not found in header\n");
-
- int i;
- if ( args->tag&GUESS_PL )
- {
- args->pl2p = (double*) calloc(256,sizeof(double));
- for (i=0; i<256; i++) args->pl2p[i] = pow(10., -i/10.);
- }
- if ( args->tag&GUESS_PL || args->tag&GUESS_GL || args->tag&GUESS_GT )
- args->tmpf = (double*) malloc(sizeof(*args->tmpf)*3*args->nsample);
-
- if ( args->verbose )
- {
- fprintf(bcftools_stdout, "# This file was produced by: bcftools +guess-ploidy(%s+htslib-%s)\n", bcftools_version(),hts_version());
- fprintf(bcftools_stdout, "# The command line was:\tbcftools +%s", args->argv[0]);
- for (i=1; i<args->argc; i++)
- fprintf(bcftools_stdout, " %s",args->argv[i]);
- fprintf(bcftools_stdout, "\n");
- fprintf(bcftools_stdout, "# [1]SEX\t[2]Sample\t[3]Predicted sex\t[4]log P(Haploid)/nSites\t[5]log P(Diploid)/nSites\t[6]nSites\t[7]Score: F < 0 < M ($4-$5)\n");
- if ( args->verbose>1 )
- fprintf(bcftools_stdout, "# [1]DBG\t[2]Chr\t[3]Pos\t[4]Sample\t[5]AF\t[6]pRR\t[7]pRA\t[8]pAA\t[9]P(Haploid)\t[10]P(Diploid)\n");
- }
-
- process_region_guess(args);
-
- for (i=0; i<args->nsample; i++)
- {
- double phap = args->stats.counts[i].ncount ? args->stats.counts[i].phap / args->stats.counts[i].ncount : 0.5;
- double pdip = args->stats.counts[i].ncount ? args->stats.counts[i].pdip / args->stats.counts[i].ncount : 0.5;
- char predicted_sex = 'U';
- if (phap>pdip) predicted_sex = 'M';
- else if (phap<pdip) predicted_sex = 'F';
- if ( args->verbose )
- {
- fprintf(bcftools_stdout, "SEX\t%s\t%c\t%f\t%f\t%"PRId64"\t%f\n", args->hdr->samples[i],predicted_sex,
- phap,pdip,args->stats.counts[i].ncount,phap-pdip);
- }
- else
- fprintf(bcftools_stdout, "%s\t%c\n", args->hdr->samples[i],predicted_sex);
- }
-
- if ( args->filter )
- filter_destroy(args->filter);
-
- bcf_sr_destroy(args->sr);
- free(args->pl2p);
- free(args->tmpf);
- free(args->counts);
- free(args->stats.counts);
- free(args->arr);
- free(args->farr);
- free(args->af);
- free(args);
- return 0;
-}
+++ /dev/null
-/* plugins/impute-info.c -- adds info metrics to a VCF file.
-
- Copyright (C) 2015-2016 Genome Research Ltd.
-
- Author: Shane McCarthy <sm15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-
-/*
-
-Marchini & Howie, Nature Genetics, 11 (July 2010)
-
-Let G_ij in {0,1,2} be the genotype of the ith individual at the
-jth SNP in a study cohort of N samples. Let
-
- p_ijk = P(G_ij = k | H,G)
-
-be the probability that the genotype at the jth SNP of the ith
-individual is k.
-
-Let the expected allele dosage for the genotype at the jth SNP
-of the ith individual be
-
- e_ij = p_ij1 + 2 * p_ij2
-
-and define
-
- f_ij = p_ij1 + 4 * p_ij2
-
-Let theta_j denote the (unknown) population allele frequency of the jth SNP
-with:
-
- theta_j = SUM[i=1..N] e_ij / 2 * N
-
-The IMPUTE2 information measure is then:
-
- if theta_j in (0,1):
- I(theta_j) = 1 - SUM[i=1..N](f_ij - e_ij^2) / 2 * N * theta_j * (1 - theta_j)
- else:
- I(theta_j) = 1
-
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-#include <math.h>
-#include <getopt.h>
-
-const char *about(void)
-{
- return "Add imputation information metrics to the INFO field based on selected FORMAT tags.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Add imputation information metrics to the INFO field based\n"
- " on selected FORMAT tags. Only the IMPUTE2 INFO metric from\n"
- " FORMAT/GP tags is currently available.\n"
- "Usage: bcftools +impute-info [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- // "Plugin options:\n"
- // " -i, --info <list> information metrics to add [INFO]\n" // [BEAGLE_R2,MACH_R2]
- // " -t, --tags <tag> VCF tags to determine the information from [GP]\n"
- // "\n"
- "Example:\n"
- " bcftools +impute-info in.vcf\n"
- "\n";
-}
-
-bcf_hdr_t *in_hdr = NULL, *out_hdr = NULL;
-int gp_type = BCF_HT_REAL;
-uint8_t *buf = NULL;
-int nbuf = 0; // NB: number of elements, not bytes
-int nrec = 0, nskip_gp = 0, nskip_dip = 0;
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- in_hdr = in;
- out_hdr = out;
- bcf_hdr_append(out_hdr,"##INFO=<ID=INFO,Number=1,Type=Float,Description=\"IMPUTE2 info score\">");
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int nval = 0, i, j, nret = bcf_get_format_values(in_hdr,rec,"GP",(void**)&buf,&nbuf,gp_type);
- if ( nret<0 )
- {
- if (!nskip_gp) fprintf(stderr, "[impute-info.c] Warning: info tag not added to sites without GP tag\n");
- nskip_gp++;
- return rec; // require FORMAT/GP tag, return site unchanged
- }
-
- nret /= rec->n_sample;
- if ( nret != 3 )
- {
- if (!nskip_dip) fprintf(stderr, "[impute-info.c] Warning: info tag not added to sites that are not biallelic diploid\n");
- nskip_dip++;
- return rec; // require biallelic diploid, return site unchanged
- }
-
- double esum = 0, e2sum = 0, fsum = 0;
- #define BRANCH(type_t,is_missing,is_vector_end) \
- { \
- type_t *ptr = (type_t*) buf; \
- for (i=0; i<rec->n_sample; i++) \
- { \
- double vals[3] = {0,0,0}; \
- for (j=0; j<nret; j++) \
- { \
- if ( is_missing || is_vector_end ) break; \
- vals[j] = ptr[j]; \
- } \
- double norm = vals[0]+vals[1]+vals[2]; \
- if ( norm ) for (j=0; j<3; j++) vals[j] /= norm; \
- esum += vals[1] + 2*vals[2]; \
- e2sum += (vals[1] + 2*vals[2]) * (vals[1] + 2*vals[2]); \
- fsum += vals[1] + 4*vals[2]; \
- ptr += nret; \
- nval++; \
- } \
- }
- switch (gp_type)
- {
- case BCF_HT_INT: BRANCH(int32_t,ptr[j]==bcf_int32_missing,ptr[j]==bcf_int32_vector_end); break;
- case BCF_HT_REAL: BRANCH(float,bcf_float_is_missing(ptr[j]),bcf_float_is_vector_end(ptr[j])); break;
- }
- #undef BRANCH
-
- double theta = esum / (2 * (double)nval);
- float info = (theta>0 && theta<1) ? (float)(1 - (fsum - e2sum) / (2 * (double)nval * theta * (1.0 - theta))) : 1;
-
- bcf_update_info_float(out_hdr, rec, "INFO", &info, 1);
- nrec++;
- return rec;
-}
-
-
-void destroy(void)
-{
- fprintf(stderr,"Lines total/info-added/unchanged-no-tag/unchanged-not-biallelic-diploid:\t%d/%d/%d/%d\n", nrec+nskip_gp+nskip_dip, nrec, nskip_gp, nskip_dip);
- free(buf);
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugins/impute-info.c -- adds info metrics to a VCF file.
-
- Copyright (C) 2015-2016 Genome Research Ltd.
-
- Author: Shane McCarthy <sm15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-
-/*
-
-Marchini & Howie, Nature Genetics, 11 (July 2010)
-
-Let G_ij in {0,1,2} be the genotype of the ith individual at the
-jth SNP in a study cohort of N samples. Let
-
- p_ijk = P(G_ij = k | H,G)
-
-be the probability that the genotype at the jth SNP of the ith
-individual is k.
-
-Let the expected allele dosage for the genotype at the jth SNP
-of the ith individual be
-
- e_ij = p_ij1 + 2 * p_ij2
-
-and define
-
- f_ij = p_ij1 + 4 * p_ij2
-
-Let theta_j denote the (unknown) population allele frequency of the jth SNP
-with:
-
- theta_j = SUM[i=1..N] e_ij / 2 * N
-
-The IMPUTE2 information measure is then:
-
- if theta_j in (0,1):
- I(theta_j) = 1 - SUM[i=1..N](f_ij - e_ij^2) / 2 * N * theta_j * (1 - theta_j)
- else:
- I(theta_j) = 1
-
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-#include <math.h>
-#include <getopt.h>
-
-const char *about(void)
-{
- return "Add imputation information metrics to the INFO field based on selected FORMAT tags.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Add imputation information metrics to the INFO field based\n"
- " on selected FORMAT tags. Only the IMPUTE2 INFO metric from\n"
- " FORMAT/GP tags is currently available.\n"
- "Usage: bcftools +impute-info [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- // "Plugin options:\n"
- // " -i, --info <list> information metrics to add [INFO]\n" // [BEAGLE_R2,MACH_R2]
- // " -t, --tags <tag> VCF tags to determine the information from [GP]\n"
- // "\n"
- "Example:\n"
- " bcftools +impute-info in.vcf\n"
- "\n";
-}
-
-bcf_hdr_t *in_hdr = NULL, *out_hdr = NULL;
-int gp_type = BCF_HT_REAL;
-uint8_t *buf = NULL;
-int nbuf = 0; // NB: number of elements, not bytes
-int nrec = 0, nskip_gp = 0, nskip_dip = 0;
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- in_hdr = in;
- out_hdr = out;
- bcf_hdr_append(out_hdr,"##INFO=<ID=INFO,Number=1,Type=Float,Description=\"IMPUTE2 info score\">");
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int nval = 0, i, j, nret = bcf_get_format_values(in_hdr,rec,"GP",(void**)&buf,&nbuf,gp_type);
- if ( nret<0 )
- {
- if (!nskip_gp) fprintf(bcftools_stderr, "[impute-info.c] Warning: info tag not added to sites without GP tag\n");
- nskip_gp++;
- return rec; // require FORMAT/GP tag, return site unchanged
- }
-
- nret /= rec->n_sample;
- if ( nret != 3 )
- {
- if (!nskip_dip) fprintf(bcftools_stderr, "[impute-info.c] Warning: info tag not added to sites that are not biallelic diploid\n");
- nskip_dip++;
- return rec; // require biallelic diploid, return site unchanged
- }
-
- double esum = 0, e2sum = 0, fsum = 0;
- #define BRANCH(type_t,is_missing,is_vector_end) \
- { \
- type_t *ptr = (type_t*) buf; \
- for (i=0; i<rec->n_sample; i++) \
- { \
- double vals[3] = {0,0,0}; \
- for (j=0; j<nret; j++) \
- { \
- if ( is_missing || is_vector_end ) break; \
- vals[j] = ptr[j]; \
- } \
- double norm = vals[0]+vals[1]+vals[2]; \
- if ( norm ) for (j=0; j<3; j++) vals[j] /= norm; \
- esum += vals[1] + 2*vals[2]; \
- e2sum += (vals[1] + 2*vals[2]) * (vals[1] + 2*vals[2]); \
- fsum += vals[1] + 4*vals[2]; \
- ptr += nret; \
- nval++; \
- } \
- }
- switch (gp_type)
- {
- case BCF_HT_INT: BRANCH(int32_t,ptr[j]==bcf_int32_missing,ptr[j]==bcf_int32_vector_end); break;
- case BCF_HT_REAL: BRANCH(float,bcf_float_is_missing(ptr[j]),bcf_float_is_vector_end(ptr[j])); break;
- }
- #undef BRANCH
-
- double theta = esum / (2 * (double)nval);
- float info = (theta>0 && theta<1) ? (float)(1 - (fsum - e2sum) / (2 * (double)nval * theta * (1.0 - theta))) : 1;
-
- bcf_update_info_float(out_hdr, rec, "INFO", &info, 1);
- nrec++;
- return rec;
-}
-
-
-void destroy(void)
-{
- fprintf(bcftools_stderr,"Lines total/info-added/unchanged-no-tag/unchanged-not-biallelic-diploid:\t%d/%d/%d/%d\n", nrec+nskip_gp+nskip_dip, nrec, nskip_gp, nskip_dip);
- free(buf);
-}
+++ /dev/null
-/*
- Copyright (C) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <stdint.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include <inttypes.h>
-#include <unistd.h>
-#include <errno.h>
-#include "bcftools.h"
-#include "smpl_ilist.h"
-
-typedef struct
-{
- int argc, output_type, regions_is_file, targets_is_file;
- char **argv, *output_fname, *regions_list, *targets_list;
- int32_t *arr_a, narr_a, *arr_b, narr_b;
- bcf_srs_t *sr;
- bcf_hdr_t *hdr_a, *hdr_b;
- htsFile *out_fh;
-}
-args_t;
-
-const char *about(void)
-{
- return "Compare two files and set non-identical genotypes to missing.\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Compare two files and set non-identical genotypes in the first file to missing.\n"
- "\n"
- "Usage: bcftools +isecGT <A.bcf> <B.bcf> [Plugin Options]\n"
- "Plugin options:\n"
- " -o, --output <file> write output to a file [standard output]\n"
- " -O, --output-type <b|u|z|v> 'b' compressed BCF; 'u' uncompressed BCF; 'z' compressed VCF; 'v' uncompressed VCF [v]\n"
- " -r, --regions <region> restrict to comma-separated list of regions\n"
- " -R, --regions-file <file> restrict to regions listed in a file\n"
- " -t, --targets <region> similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n"
- "\n";
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->output_fname = "-";
- args->output_type = FT_VCF;
- static struct option loptions[] =
- {
- {"regions",required_argument,NULL,'r'},
- {"regions-file",required_argument,NULL,'R'},
- {"targets",required_argument,NULL,'t'},
- {"targets-file",required_argument,NULL,'T'},
- {"output",required_argument,NULL,'o'},
- {"output-type",required_argument,NULL,'O'},
- {NULL,0,NULL,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "o:O:r:R:t:T:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'o': args->output_fname = optarg; break;
- case 'O':
- switch (optarg[0]) {
- case 'b': args->output_type = FT_BCF_GZ; break;
- case 'u': args->output_type = FT_BCF; break;
- case 'z': args->output_type = FT_VCF_GZ; break;
- case 'v': args->output_type = FT_VCF; break;
- default: error("The output type \"%s\" not recognised\n", optarg);
- }
- break;
- case 'r': args->regions_list = optarg; break;
- case 'R': args->regions_list = optarg; args->regions_is_file = 1; break;
- case 't': args->targets_list = optarg; break;
- case 'T': args->targets_list = optarg; args->targets_is_file = 1; break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
-
- if ( optind+2!=argc ) error("%s",usage_text());
-
- args->sr = bcf_sr_init();
- args->sr->require_index = 1;
- if ( args->regions_list )
- {
- if ( bcf_sr_set_regions(args->sr, args->regions_list, args->regions_is_file)<0 )
- error("Failed to read the regions: %s\n", args->regions_list);
- }
- if ( args->targets_list )
- {
- if ( bcf_sr_set_targets(args->sr, args->targets_list, args->targets_is_file, 0)<0 )
- error("Failed to read the targets: %s\n", args->targets_list);
- args->sr->collapse |= COLLAPSE_BOTH;
- }
- if ( !bcf_sr_add_reader(args->sr,argv[optind]) ) error("Error opening %s: %s\n", argv[optind],bcf_sr_strerror(args->sr->errnum));
- if ( !bcf_sr_add_reader(args->sr,argv[optind+1]) ) error("Error opening %s: %s\n", argv[optind+1],bcf_sr_strerror(args->sr->errnum));
- args->hdr_a = bcf_sr_get_header(args->sr,0);
- args->hdr_b = bcf_sr_get_header(args->sr,1);
- smpl_ilist_t *smpl = smpl_ilist_map(args->hdr_a, args->hdr_b, SMPL_STRICT);
- args->out_fh = hts_open(args->output_fname, hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
- bcf_hdr_write(args->out_fh, args->hdr_a);
-
- while ( bcf_sr_next_line(args->sr) )
- {
- if ( !bcf_sr_has_line(args->sr,0) ) continue;
- if ( !bcf_sr_has_line(args->sr,1) )
- {
- bcf_write(args->out_fh, args->hdr_a, bcf_sr_get_line(args->sr,0));
- continue;
- }
-
- bcf1_t *line_a = bcf_sr_get_line(args->sr,0);
- bcf1_t *line_b = bcf_sr_get_line(args->sr,1);
- int ngt_a = bcf_get_genotypes(args->hdr_a, line_a, &args->arr_a, &args->narr_a);
- int ngt_b = bcf_get_genotypes(args->hdr_b, line_b, &args->arr_b, &args->narr_b);
- assert( ngt_a==ngt_b ); // todo
- ngt_a /= smpl->n;
- ngt_b /= smpl->n;
- int i, j, dirty = 0;
- for (i=0; i<smpl->n; i++)
- {
- int32_t *a = args->arr_a + i*ngt_a;
- int32_t *b = args->arr_b + smpl->idx[i]*ngt_b;
- for (j=0; j<ngt_a; j++)
- if ( a[j]!=b[j] ) break;
- if ( j<ngt_a )
- {
- dirty = 1;
- for (j=0; j<ngt_a; j++) a[j] = bcf_gt_missing;
- }
- }
- if ( dirty ) bcf_update_genotypes(args->hdr_a, line_a, args->arr_a, ngt_a*smpl->n);
- bcf_write(args->out_fh, args->hdr_a, line_a);
- }
-
- if ( hts_close(args->out_fh)!=0 ) error("Close failed: %s\n",args->output_fname);
- smpl_ilist_destroy(smpl);
- bcf_sr_destroy(args->sr);
- free(args->arr_a);
- free(args->arr_b);
- free(args);
- return 0;
-}
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/*
- Copyright (C) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <stdint.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include <inttypes.h>
-#include <unistd.h>
-#include <errno.h>
-#include "bcftools.h"
-#include "smpl_ilist.h"
-
-typedef struct
-{
- int argc, output_type, regions_is_file, targets_is_file;
- char **argv, *output_fname, *regions_list, *targets_list;
- int32_t *arr_a, narr_a, *arr_b, narr_b;
- bcf_srs_t *sr;
- bcf_hdr_t *hdr_a, *hdr_b;
- htsFile *out_fh;
-}
-args_t;
-
-const char *about(void)
-{
- return "Compare two files and set non-identical genotypes to missing.\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Compare two files and set non-identical genotypes in the first file to missing.\n"
- "\n"
- "Usage: bcftools +isecGT <A.bcf> <B.bcf> [Plugin Options]\n"
- "Plugin options:\n"
- " -o, --output <file> write output to a file [standard output]\n"
- " -O, --output-type <b|u|z|v> 'b' compressed BCF; 'u' uncompressed BCF; 'z' compressed VCF; 'v' uncompressed VCF [v]\n"
- " -r, --regions <region> restrict to comma-separated list of regions\n"
- " -R, --regions-file <file> restrict to regions listed in a file\n"
- " -t, --targets <region> similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n"
- "\n";
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->output_fname = "-";
- args->output_type = FT_VCF;
- static struct option loptions[] =
- {
- {"regions",required_argument,NULL,'r'},
- {"regions-file",required_argument,NULL,'R'},
- {"targets",required_argument,NULL,'t'},
- {"targets-file",required_argument,NULL,'T'},
- {"output",required_argument,NULL,'o'},
- {"output-type",required_argument,NULL,'O'},
- {NULL,0,NULL,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "o:O:r:R:t:T:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'o': args->output_fname = optarg; break;
- case 'O':
- switch (optarg[0]) {
- case 'b': args->output_type = FT_BCF_GZ; break;
- case 'u': args->output_type = FT_BCF; break;
- case 'z': args->output_type = FT_VCF_GZ; break;
- case 'v': args->output_type = FT_VCF; break;
- default: error("The output type \"%s\" not recognised\n", optarg);
- }
- break;
- case 'r': args->regions_list = optarg; break;
- case 'R': args->regions_list = optarg; args->regions_is_file = 1; break;
- case 't': args->targets_list = optarg; break;
- case 'T': args->targets_list = optarg; args->targets_is_file = 1; break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
-
- if ( optind+2!=argc ) error("%s",usage_text());
-
- args->sr = bcf_sr_init();
- args->sr->require_index = 1;
- if ( args->regions_list )
- {
- if ( bcf_sr_set_regions(args->sr, args->regions_list, args->regions_is_file)<0 )
- error("Failed to read the regions: %s\n", args->regions_list);
- }
- if ( args->targets_list )
- {
- if ( bcf_sr_set_targets(args->sr, args->targets_list, args->targets_is_file, 0)<0 )
- error("Failed to read the targets: %s\n", args->targets_list);
- args->sr->collapse |= COLLAPSE_BOTH;
- }
- if ( !bcf_sr_add_reader(args->sr,argv[optind]) ) error("Error opening %s: %s\n", argv[optind],bcf_sr_strerror(args->sr->errnum));
- if ( !bcf_sr_add_reader(args->sr,argv[optind+1]) ) error("Error opening %s: %s\n", argv[optind+1],bcf_sr_strerror(args->sr->errnum));
- args->hdr_a = bcf_sr_get_header(args->sr,0);
- args->hdr_b = bcf_sr_get_header(args->sr,1);
- smpl_ilist_t *smpl = smpl_ilist_map(args->hdr_a, args->hdr_b, SMPL_STRICT);
- args->out_fh = hts_open(args->output_fname, hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
- bcf_hdr_write(args->out_fh, args->hdr_a);
-
- while ( bcf_sr_next_line(args->sr) )
- {
- if ( !bcf_sr_has_line(args->sr,0) ) continue;
- if ( !bcf_sr_has_line(args->sr,1) )
- {
- bcf_write(args->out_fh, args->hdr_a, bcf_sr_get_line(args->sr,0));
- continue;
- }
-
- bcf1_t *line_a = bcf_sr_get_line(args->sr,0);
- bcf1_t *line_b = bcf_sr_get_line(args->sr,1);
- int ngt_a = bcf_get_genotypes(args->hdr_a, line_a, &args->arr_a, &args->narr_a);
- int ngt_b = bcf_get_genotypes(args->hdr_b, line_b, &args->arr_b, &args->narr_b);
- assert( ngt_a==ngt_b ); // todo
- ngt_a /= smpl->n;
- ngt_b /= smpl->n;
- int i, j, dirty = 0;
- for (i=0; i<smpl->n; i++)
- {
- int32_t *a = args->arr_a + i*ngt_a;
- int32_t *b = args->arr_b + smpl->idx[i]*ngt_b;
- for (j=0; j<ngt_a; j++)
- if ( a[j]!=b[j] ) break;
- if ( j<ngt_a )
- {
- dirty = 1;
- for (j=0; j<ngt_a; j++) a[j] = bcf_gt_missing;
- }
- }
- if ( dirty ) bcf_update_genotypes(args->hdr_a, line_a, args->arr_a, ngt_a*smpl->n);
- bcf_write(args->out_fh, args->hdr_a, line_a);
- }
-
- if ( hts_close(args->out_fh)!=0 ) error("Close failed: %s\n",args->output_fname);
- smpl_ilist_destroy(smpl);
- bcf_sr_destroy(args->sr);
- free(args->arr_a);
- free(args->arr_b);
- free(args);
- return 0;
-}
-
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2015 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <errno.h>
-#include <ctype.h>
-#include <unistd.h> // for isatty
-#include "bcftools.h"
-#include "regidx.h"
-
-#define MODE_COUNT 1
-#define MODE_LIST_GOOD 2
-#define MODE_LIST_BAD 4
-#define MODE_DELETE 8
-
-typedef struct
-{
- int nok, nbad;
- int imother,ifather,ichild;
-}
-trio_t;
-
-typedef struct
-{
- int mpl, fpl, cpl; // ploidies - mother, father, child
- int mal, fal; // expect an allele from mother and father
-}
-rule_t;
-
-typedef struct _args_t
-{
- regidx_t *rules;
- regitr_t *itr, *itr_ori;
- bcf_hdr_t *hdr;
- htsFile *out_fh;
- int32_t *gt_arr;
- int mode;
- int ngt_arr, nrec;
- trio_t *trios;
- int ntrios;
- int output_type;
- char *output_fname;
- bcf_srs_t *sr;
-}
-args_t;
-
-static args_t args;
-static int parse_rules(const char *line, char **chr_beg, char **chr_end, uint32_t *beg, uint32_t *end, void *payload, void *usr);
-static bcf1_t *process(bcf1_t *rec);
-
-const char *about(void)
-{
- return "Count Mendelian consistent / inconsistent genotypes.\n";
-}
-
-typedef struct
-{
- const char *alias, *about, *rules;
-}
-rules_predef_t;
-
-static rules_predef_t rules_predefs[] =
-{
- { .alias = "GRCh37",
- .about = "Human Genome reference assembly GRCh37 / hg19, both chr naming conventions",
- .rules =
- " X:1-60000 M/M + F > M\n"
- " X:1-60000 M/M + F > M/F\n"
- " X:2699521-154931043 M/M + F > M\n"
- " X:2699521-154931043 M/M + F > M/F\n"
- " Y:1-59373566 . + F > F\n"
- " MT:1-16569 M + F > M\n"
- "\n"
- " chrX:1-60000 M/M + F > M\n"
- " chrX:1-60000 M/M + F > M/F\n"
- " chrX:2699521-154931043 M/M + F > M\n"
- " chrX:2699521-154931043 M/M + F > M/F\n"
- " chrY:1-59373566 . + F > F\n"
- " chrM:1-16569 M + F > M\n"
- },
- { .alias = "GRCh38",
- .about = "Human Genome reference assembly GRCh38 / hg38, both chr naming conventions",
- .rules =
- " X:1-9999 M/M + F > M\n"
- " X:1-9999 M/M + F > M/F\n"
- " X:2781480-155701381 M/M + F > M\n"
- " X:2781480-155701381 M/M + F > M/F\n"
- " Y:1-57227415 . + F > F\n"
- " MT:1-16569 M + F > M\n"
- "\n"
- " chrX:1-9999 M/M + F > M\n"
- " chrX:1-9999 M/M + F > M/F\n"
- " chrX:2781480-155701381 M/M + F > M\n"
- " chrX:2781480-155701381 M/M + F > M/F\n"
- " chrY:1-57227415 . + F > F\n"
- " chrM:1-16569 M + F > M\n"
- },
- {
- .alias = NULL,
- .about = NULL,
- .rules = NULL,
- }
-};
-
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Count Mendelian consistent / inconsistent genotypes.\n"
- "Usage: bcftools +mendelian [Options]\n"
- "Options:\n"
- " -c, --count count the number of consistent sites\n"
- " -d, --delete delete inconsistent genotypes (set to \"./.\")\n"
- " -l, --list [+x] list consistent (+) or inconsistent (x) sites\n"
- " -o, --output <file> write output to a file [standard output]\n"
- " -O, --output-type <type> 'b' compressed BCF; 'u' uncompressed BCF; 'z' compressed VCF; 'v' uncompressed VCF [v]\n"
- " -r, --rules <assembly>[?] predefined rules, 'list' to print available settings, append '?' for details\n"
- " -R, --rules-file <file> inheritance rules, see example below\n"
- " -t, --trio <m,f,c> names of mother, father and the child\n"
- " -T, --trio-file <file> list of trios, one per line\n"
- "\n"
- "Example:\n"
- " # Default inheritance patterns, override with -r\n"
- " # region mothernal_ploidy + paternal > offspring\n"
- " X:1-60000 M/M + F > M\n"
- " X:1-60000 M/M + F > M/F\n"
- " X:2699521-154931043 M/M + F > M\n"
- " X:2699521-154931043 M/M + F > M/F\n"
- " Y:1-59373566 . + F > F\n"
- " MT:1-16569 M + F > M\n"
- "\n"
- " bcftools +mendelian in.vcf -t Mother,Father,Child -c\n"
- "\n";
-}
-
-regidx_t *init_rules(args_t *args, char *alias)
-{
- const rules_predef_t *rules = rules_predefs;
- if ( !alias ) alias = "GRCh37";
-
- int detailed = 0, len = strlen(alias);
- if ( alias[len-1]=='?' ) { detailed = 1; alias[len-1] = 0; }
-
- while ( rules->alias && strcasecmp(alias,rules->alias) ) rules++;
-
- if ( !rules->alias )
- {
- fprintf(stderr,"\nPRE-DEFINED INHERITANCE RULES\n\n");
- fprintf(stderr," * Columns are: CHROM:BEG-END MATERNAL_PLOIDY + PATERNAL_PLOIDY > OFFSPRING\n");
- fprintf(stderr," * Coordinates are 1-based inclusive.\n\n");
- rules = rules_predefs;
- while ( rules->alias )
- {
- fprintf(stderr,"%s\n .. %s\n\n", rules->alias,rules->about);
- if ( detailed )
- fprintf(stderr,"%s\n", rules->rules);
- rules++;
- }
- fprintf(stderr,"Run as --rules <alias> (e.g. --rules GRCh37).\n");
- fprintf(stderr,"To see the detailed ploidy definition, append a question mark (e.g. --rules GRCh37?).\n");
- fprintf(stderr,"\n");
- exit(-1);
- }
- else if ( detailed )
- {
- fprintf(stderr,"%s", rules->rules);
- exit(-1);
- }
- return regidx_init_string(rules->rules, parse_rules, NULL, sizeof(rule_t), &args);
-}
-
-static int parse_rules(const char *line, char **chr_beg, char **chr_end, uint32_t *beg, uint32_t *end, void *payload, void *usr)
-{
- // e.g. "Y:1-59373566 . + F > . # daugther"
-
- // eat any leading spaces
- char *ss = (char*) line;
- while ( *ss && isspace(*ss) ) ss++;
- if ( !*ss ) return -1; // skip empty lines
-
- // chromosome name, beg, end
- char *tmp, *se = ss;
- while ( se[1] && !isspace(se[1]) ) se++;
- while ( se > ss && isdigit(*se) ) se--;
- if ( *se!='-' ) error("Could not parse the region: %s\n", line);
- *end = strtol(se+1, &tmp, 10) - 1;
- if ( tmp==se+1 ) error("Could not parse the region:%s\n",line);
- while ( se > ss && *se!=':' ) se--;
- *beg = strtol(se+1, &tmp, 10) - 1;
- if ( tmp==se+1 ) error("Could not parse the region:%s\n",line);
-
- *chr_beg = ss;
- *chr_end = se-1;
-
- // skip region
- while ( *ss && !isspace(*ss) ) ss++;
- while ( *ss && isspace(*ss) ) ss++;
-
- rule_t *rule = (rule_t*) payload;
- memset(rule, 0, sizeof(rule_t));
-
- // mothernal ploidy
- se = ss;
- while ( *se && !isspace(*se) ) se++;
- int err = 0;
- if ( se - ss == 1 )
- {
- if ( *ss=='M' ) rule->mpl = 1;
- else if ( *ss=='.' ) rule->mpl = 0;
- else err = 1;
- }
- else if ( se - ss == 3 )
- {
- if ( !strncmp(ss,"M/M",3) ) rule->mpl = 2;
- else err = 1;
- }
- else err = 1;
- if ( err ) error("Could not parse the mothernal ploidy, only \"M\", \"M/M\" and \".\" currently supported: %s\n",line);
-
- // skip "+"
- while ( *se && isspace(*se) ) se++;
- if ( *se != '+' ) error("Could not parse the line: %s\n",line);
- se++;
- while ( *se && isspace(*se) ) se++;
-
- // paternal ploidy
- ss = se;
- while ( *se && !isspace(*se) ) se++;
- if ( se - ss == 1 )
- {
- if ( *ss=='F' ) rule->fpl = 1;
- else err = 1;
- }
- else err = 1;
- if ( err ) error("Could not parse the paternal ploidy, only \"F\" is currently supported: %s [%s]\n",line, ss);
-
- // skip ">"
- while ( *se && isspace(*se) ) se++;
- if ( *se != '>' ) error("Could not parse the line: %s\n",line);
- se++;
- while ( *se && isspace(*se) ) se++;
-
- // ploidy in offspring
- ss = se;
- while ( *se && !isspace(*se) ) se++;
- if ( se - ss == 3 )
- {
- if ( !strncmp(ss,"M/F",3) ) { rule->cpl = 2; rule->fal = 1; rule->mal = 1; }
- else err = 1;
- }
- else if ( se - ss == 1 )
- {
- if ( *ss=='F' ) { rule->cpl = 1; rule->fal = 1; }
- else if ( *ss=='M' ) { rule->cpl = 1; rule->mal = 1; }
- else err = 1;
- }
- else err = 1;
- if ( err ) error("Could not parse the offspring's ploidy, only \"M\", \"F\" or \"M/F\" is currently supported: %s\n",line);
-
- return 0;
-}
-
-int run(int argc, char **argv)
-{
- char *trio_samples = NULL, *trio_file = NULL, *rules_fname = NULL, *rules_string = NULL;
- memset(&args,0,sizeof(args_t));
- args.mode = 0;
- args.output_fname = "-";
-
- static struct option loptions[] =
- {
- {"trio",1,0,'t'},
- {"trio-file",1,0,'T'},
- {"delete",0,0,'d'},
- {"list",1,0,'l'},
- {"count",0,0,'c'},
- {"rules",1,0,'r'},
- {"rules-file",1,0,'R'},
- {"output",required_argument,NULL,'o'},
- {"output-type",required_argument,NULL,'O'},
- {0,0,0,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?ht:T:l:cdr:R:o:O:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'o': args.output_fname = optarg; break;
- case 'O':
- switch (optarg[0]) {
- case 'b': args.output_type = FT_BCF_GZ; break;
- case 'u': args.output_type = FT_BCF; break;
- case 'z': args.output_type = FT_VCF_GZ; break;
- case 'v': args.output_type = FT_VCF; break;
- default: error("The output type \"%s\" not recognised\n", optarg);
- };
- break;
- case 'R': rules_fname = optarg; break;
- case 'r': rules_string = optarg; break;
- case 'd': args.mode |= MODE_DELETE; break;
- case 'c': args.mode |= MODE_COUNT; break;
- case 'l':
- if ( !strcmp("+",optarg) ) args.mode |= MODE_LIST_GOOD;
- else if ( !strcmp("x",optarg) ) args.mode |= MODE_LIST_BAD;
- else error("The argument not recognised: --list %s\n", optarg);
- break;
- case 't': trio_samples = optarg; break;
- case 'T': trio_file = optarg; break;
- case 'h':
- case '?':
- default: error("%s",usage()); break;
- }
- }
- if ( rules_fname )
- args.rules = regidx_init(rules_fname, parse_rules, NULL, sizeof(rule_t), &args);
- else
- args.rules = init_rules(&args, rules_string);
- if ( !args.rules ) return -1;
- args.itr = regitr_init(args.rules);
- args.itr_ori = regitr_init(args.rules);
-
- char *fname = NULL;
- if ( optind>=argc || argv[optind][0]=='-' )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) fname = "-"; // reading from stdin
- else error("%s",usage());
- }
- else
- fname = argv[optind];
-
- if ( !trio_samples && !trio_file ) error("Expected the -t/T option\n");
- if ( !args.mode ) error("Expected one of the -c, -d or -l options\n");
- if ( args.mode&MODE_DELETE && !(args.mode&(MODE_LIST_GOOD|MODE_LIST_BAD)) ) args.mode |= MODE_LIST_GOOD|MODE_LIST_BAD;
-
- args.sr = bcf_sr_init();
- if ( !bcf_sr_add_reader(args.sr, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args.sr->errnum));
- args.hdr = bcf_sr_get_header(args.sr, 0);
- args.out_fh = hts_open(args.output_fname,hts_bcf_wmode(args.output_type));
- if ( args.out_fh == NULL ) error("Can't write to \"%s\": %s\n", args.output_fname, strerror(errno));
- bcf_hdr_write(args.out_fh, args.hdr);
-
-
- int i, n = 0;
- char **list;
- if ( trio_samples )
- {
- args.ntrios = 1;
- args.trios = (trio_t*) calloc(1,sizeof(trio_t));
- list = hts_readlist(trio_samples, 0, &n);
- if ( n!=3 ) error("Expected three sample names with -t\n");
- args.trios[0].imother = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[0]);
- args.trios[0].ifather = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[1]);
- args.trios[0].ichild = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[2]);
- for (i=0; i<n; i++) free(list[i]);
- free(list);
- }
- if ( trio_file )
- {
- list = hts_readlist(trio_file, 1, &n);
- args.ntrios = n;
- args.trios = (trio_t*) calloc(n,sizeof(trio_t));
- for (i=0; i<n; i++)
- {
- char *ss = list[i], *se;
- se = strchr(ss, ',');
- if ( !se ) error("Could not parse %s: %s\n",trio_file, ss);
- *se = 0;
- args.trios[i].imother = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, ss);
- if ( args.trios[i].imother<0 ) error("No such sample: \"%s\"\n", ss);
- ss = ++se;
- se = strchr(ss, ',');
- if ( !se ) error("Could not parse %s\n",trio_file);
- *se = 0;
- args.trios[i].ifather = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, ss);
- if ( args.trios[i].ifather<0 ) error("No such sample: \"%s\"\n", ss);
- ss = ++se;
- if ( *ss=='\0' ) error("Could not parse %s\n",trio_file);
- args.trios[i].ichild = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, ss);
- if ( args.trios[i].ichild<0 ) error("No such sample: \"%s\"\n", ss);
- free(list[i]);
- }
- free(list);
- }
-
- while ( bcf_sr_next_line(args.sr) )
- {
- bcf1_t *line = bcf_sr_get_line(args.sr,0);
- line = process(line);
- if ( line )
- {
- if ( line->errcode ) error("TODO: Unchecked error (%d), exiting\n",line->errcode);
- bcf_write1(args.out_fh, args.hdr, line);
- }
- }
-
-
- fprintf(stderr,"# [1]nOK\t[2]nBad\t[3]nSkipped\t[4]Trio\n");
- for (i=0; i<args.ntrios; i++)
- {
- trio_t *trio = &args.trios[i];
- fprintf(stderr,"%d\t%d\t%d\t%s,%s,%s\n",
- trio->nok,trio->nbad,args.nrec-(trio->nok+trio->nbad),
- bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, trio->imother),
- bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, trio->ifather),
- bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, trio->ichild)
- );
- }
- free(args.gt_arr);
- free(args.trios);
- regitr_destroy(args.itr);
- regitr_destroy(args.itr_ori);
- regidx_destroy(args.rules);
- bcf_sr_destroy(args.sr);
- if ( hts_close(args.out_fh)!=0 ) error("Error: close failed\n");
- return 0;
-}
-
-static void warn_ploidy(bcf1_t *rec)
-{
- static int warned = 0;
- if ( warned ) return;
- fprintf(stderr,"Incorrect ploidy at %s:%d, skipping the trio. (This warning is printed only once.)\n", bcf_seqname(args.hdr,rec),rec->pos+1);
- warned = 1;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- bcf1_t *dflt = args.mode&MODE_LIST_GOOD ? rec : NULL;
- args.nrec++;
-
- if ( rec->n_allele > 63 ) return dflt; // we use 64bit bitmask below
-
- int ngt = bcf_get_genotypes(args.hdr, rec, &args.gt_arr, &args.ngt_arr);
- if ( ngt<0 ) return dflt;
- if ( ngt!=2*bcf_hdr_nsamples(args.hdr) && ngt!=bcf_hdr_nsamples(args.hdr) ) return dflt;
- ngt /= bcf_hdr_nsamples(args.hdr);
-
- int itr_set = regidx_overlap(args.rules, bcf_seqname(args.hdr,rec),rec->pos,rec->pos, args.itr_ori);
-
- int i, has_bad = 0, needs_update = 0;
- for (i=0; i<args.ntrios; i++)
- {
- int32_t a,b,c,d,e,f;
- trio_t *trio = &args.trios[i];
-
- a = args.gt_arr[ngt*trio->imother];
- b = ngt==2 ? args.gt_arr[ngt*trio->imother+1] : bcf_int32_vector_end;
- c = args.gt_arr[ngt*trio->ifather];
- d = ngt==2 ? args.gt_arr[ngt*trio->ifather+1] : bcf_int32_vector_end;
- e = args.gt_arr[ngt*trio->ichild];
- f = ngt==2 ? args.gt_arr[ngt*trio->ichild+1] : bcf_int32_vector_end;
-
- // skip sites with missing data in child
- if ( bcf_gt_is_missing(e) || bcf_gt_is_missing(f) ) continue;
-
- uint64_t mother = 0, father = 0,child1,child2;
-
- int is_ok = 0;
- if ( !itr_set )
- {
- if ( f==bcf_int32_vector_end ) { warn_ploidy(rec); continue; }
-
- // All M,F,C genotypes are diploid. Missing data are considered consistent.
- child1 = 1<<bcf_gt_allele(e);
- child2 = 1<<bcf_gt_allele(f);
- mother = bcf_gt_is_missing(a) ? child1|child2 : 1<<bcf_gt_allele(a);
- mother |= bcf_gt_is_missing(b) || b==bcf_int32_vector_end ? child1|child2 : 1<<bcf_gt_allele(b);
- father = bcf_gt_is_missing(c) ? child1|child2 : 1<<bcf_gt_allele(c);
- father |= bcf_gt_is_missing(d) || d==bcf_int32_vector_end ? child1|child2 : 1<<bcf_gt_allele(d);
-
- if ( (mother&child1 && father&child2) || (mother&child2 && father&child1) ) is_ok = 1;
- }
- else
- {
- child1 = 1<<bcf_gt_allele(e);
- child2 = bcf_gt_is_missing(f) || f==bcf_int32_vector_end ? 0 : 1<<bcf_gt_allele(f);
- mother |= bcf_gt_is_missing(a) ? 0 : 1<<bcf_gt_allele(a);
- mother |= bcf_gt_is_missing(b) || b==bcf_int32_vector_end ? 0 : 1<<bcf_gt_allele(b);
- father |= bcf_gt_is_missing(c) ? 0 : 1<<bcf_gt_allele(c);
- father |= bcf_gt_is_missing(d) || d==bcf_int32_vector_end ? 0 : 1<<bcf_gt_allele(d);
-
- regitr_copy(args.itr, args.itr_ori);
- while ( !is_ok && regitr_overlap(args.itr) )
- {
- rule_t *rule = ®itr_payload(args.itr,rule_t);
- if ( child1 && child2 )
- {
- if ( !rule->mal || !rule->fal ) continue; // wrong rule (haploid), but this is a diploid GT
- if ( !mother ) mother = child1|child2;
- if ( !father ) father = child1|child2;
- if ( (mother&child1 && father&child2) || (mother&child2 && father&child1) ) is_ok = 1;
- continue;
- }
- if ( rule->mal )
- {
- if ( mother && !(child1&mother) ) continue;
- }
- if ( rule->fal )
- {
- if ( father && !(child1&father) ) continue;
- }
- is_ok = 1;
- }
- }
- if ( is_ok )
- {
- trio->nok++;
- }
- else
- {
- trio->nbad++;
- has_bad = 1;
- if ( args.mode&MODE_DELETE )
- {
- args.gt_arr[ngt*trio->imother] = bcf_gt_missing;
- if ( b!=bcf_int32_vector_end ) args.gt_arr[ngt*trio->imother+1] = bcf_gt_missing; // should be always true
- args.gt_arr[ngt*trio->ifather] = bcf_gt_missing;
- if ( d!=bcf_int32_vector_end ) args.gt_arr[ngt*trio->ifather+1] = bcf_gt_missing;
- args.gt_arr[ngt*trio->ichild] = bcf_gt_missing;
- if ( f!=bcf_int32_vector_end ) args.gt_arr[ngt*trio->ichild+1] = bcf_gt_missing;
- needs_update = 1;
- }
- }
- }
-
- if ( needs_update && bcf_update_genotypes(args.hdr,rec,args.gt_arr,ngt*bcf_hdr_nsamples(args.hdr)) )
- error("Could not update GT field at %s:%d\n", bcf_seqname(args.hdr,rec),rec->pos+1);
-
- if ( args.mode&MODE_DELETE ) return rec;
- if ( args.mode&MODE_LIST_GOOD ) return has_bad ? NULL : rec;
- if ( args.mode&MODE_LIST_BAD ) return has_bad ? rec : NULL;
-
- return NULL;
-}
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2015 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <errno.h>
-#include <ctype.h>
-#include <unistd.h> // for isatty
-#include "bcftools.h"
-#include "regidx.h"
-
-#define MODE_COUNT 1
-#define MODE_LIST_GOOD 2
-#define MODE_LIST_BAD 4
-#define MODE_DELETE 8
-
-typedef struct
-{
- int nok, nbad;
- int imother,ifather,ichild;
-}
-trio_t;
-
-typedef struct
-{
- int mpl, fpl, cpl; // ploidies - mother, father, child
- int mal, fal; // expect an allele from mother and father
-}
-rule_t;
-
-typedef struct _args_t
-{
- regidx_t *rules;
- regitr_t *itr, *itr_ori;
- bcf_hdr_t *hdr;
- htsFile *out_fh;
- int32_t *gt_arr;
- int mode;
- int ngt_arr, nrec;
- trio_t *trios;
- int ntrios;
- int output_type;
- char *output_fname;
- bcf_srs_t *sr;
-}
-args_t;
-
-static args_t args;
-static int parse_rules(const char *line, char **chr_beg, char **chr_end, uint32_t *beg, uint32_t *end, void *payload, void *usr);
-static bcf1_t *process(bcf1_t *rec);
-
-const char *about(void)
-{
- return "Count Mendelian consistent / inconsistent genotypes.\n";
-}
-
-typedef struct
-{
- const char *alias, *about, *rules;
-}
-rules_predef_t;
-
-static rules_predef_t rules_predefs[] =
-{
- { .alias = "GRCh37",
- .about = "Human Genome reference assembly GRCh37 / hg19, both chr naming conventions",
- .rules =
- " X:1-60000 M/M + F > M\n"
- " X:1-60000 M/M + F > M/F\n"
- " X:2699521-154931043 M/M + F > M\n"
- " X:2699521-154931043 M/M + F > M/F\n"
- " Y:1-59373566 . + F > F\n"
- " MT:1-16569 M + F > M\n"
- "\n"
- " chrX:1-60000 M/M + F > M\n"
- " chrX:1-60000 M/M + F > M/F\n"
- " chrX:2699521-154931043 M/M + F > M\n"
- " chrX:2699521-154931043 M/M + F > M/F\n"
- " chrY:1-59373566 . + F > F\n"
- " chrM:1-16569 M + F > M\n"
- },
- { .alias = "GRCh38",
- .about = "Human Genome reference assembly GRCh38 / hg38, both chr naming conventions",
- .rules =
- " X:1-9999 M/M + F > M\n"
- " X:1-9999 M/M + F > M/F\n"
- " X:2781480-155701381 M/M + F > M\n"
- " X:2781480-155701381 M/M + F > M/F\n"
- " Y:1-57227415 . + F > F\n"
- " MT:1-16569 M + F > M\n"
- "\n"
- " chrX:1-9999 M/M + F > M\n"
- " chrX:1-9999 M/M + F > M/F\n"
- " chrX:2781480-155701381 M/M + F > M\n"
- " chrX:2781480-155701381 M/M + F > M/F\n"
- " chrY:1-57227415 . + F > F\n"
- " chrM:1-16569 M + F > M\n"
- },
- {
- .alias = NULL,
- .about = NULL,
- .rules = NULL,
- }
-};
-
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Count Mendelian consistent / inconsistent genotypes.\n"
- "Usage: bcftools +mendelian [Options]\n"
- "Options:\n"
- " -c, --count count the number of consistent sites\n"
- " -d, --delete delete inconsistent genotypes (set to \"./.\")\n"
- " -l, --list [+x] list consistent (+) or inconsistent (x) sites\n"
- " -o, --output <file> write output to a file [standard output]\n"
- " -O, --output-type <type> 'b' compressed BCF; 'u' uncompressed BCF; 'z' compressed VCF; 'v' uncompressed VCF [v]\n"
- " -r, --rules <assembly>[?] predefined rules, 'list' to print available settings, append '?' for details\n"
- " -R, --rules-file <file> inheritance rules, see example below\n"
- " -t, --trio <m,f,c> names of mother, father and the child\n"
- " -T, --trio-file <file> list of trios, one per line\n"
- "\n"
- "Example:\n"
- " # Default inheritance patterns, override with -r\n"
- " # region mothernal_ploidy + paternal > offspring\n"
- " X:1-60000 M/M + F > M\n"
- " X:1-60000 M/M + F > M/F\n"
- " X:2699521-154931043 M/M + F > M\n"
- " X:2699521-154931043 M/M + F > M/F\n"
- " Y:1-59373566 . + F > F\n"
- " MT:1-16569 M + F > M\n"
- "\n"
- " bcftools +mendelian in.vcf -t Mother,Father,Child -c\n"
- "\n";
-}
-
-regidx_t *init_rules(args_t *args, char *alias)
-{
- const rules_predef_t *rules = rules_predefs;
- if ( !alias ) alias = "GRCh37";
-
- int detailed = 0, len = strlen(alias);
- if ( alias[len-1]=='?' ) { detailed = 1; alias[len-1] = 0; }
-
- while ( rules->alias && strcasecmp(alias,rules->alias) ) rules++;
-
- if ( !rules->alias )
- {
- fprintf(bcftools_stderr,"\nPRE-DEFINED INHERITANCE RULES\n\n");
- fprintf(bcftools_stderr," * Columns are: CHROM:BEG-END MATERNAL_PLOIDY + PATERNAL_PLOIDY > OFFSPRING\n");
- fprintf(bcftools_stderr," * Coordinates are 1-based inclusive.\n\n");
- rules = rules_predefs;
- while ( rules->alias )
- {
- fprintf(bcftools_stderr,"%s\n .. %s\n\n", rules->alias,rules->about);
- if ( detailed )
- fprintf(bcftools_stderr,"%s\n", rules->rules);
- rules++;
- }
- fprintf(bcftools_stderr,"Run as --rules <alias> (e.g. --rules GRCh37).\n");
- fprintf(bcftools_stderr,"To see the detailed ploidy definition, append a question mark (e.g. --rules GRCh37?).\n");
- fprintf(bcftools_stderr,"\n");
- exit(-1);
- }
- else if ( detailed )
- {
- fprintf(bcftools_stderr,"%s", rules->rules);
- exit(-1);
- }
- return regidx_init_string(rules->rules, parse_rules, NULL, sizeof(rule_t), &args);
-}
-
-static int parse_rules(const char *line, char **chr_beg, char **chr_end, uint32_t *beg, uint32_t *end, void *payload, void *usr)
-{
- // e.g. "Y:1-59373566 . + F > . # daugther"
-
- // eat any leading spaces
- char *ss = (char*) line;
- while ( *ss && isspace(*ss) ) ss++;
- if ( !*ss ) return -1; // skip empty lines
-
- // chromosome name, beg, end
- char *tmp, *se = ss;
- while ( se[1] && !isspace(se[1]) ) se++;
- while ( se > ss && isdigit(*se) ) se--;
- if ( *se!='-' ) error("Could not parse the region: %s\n", line);
- *end = strtol(se+1, &tmp, 10) - 1;
- if ( tmp==se+1 ) error("Could not parse the region:%s\n",line);
- while ( se > ss && *se!=':' ) se--;
- *beg = strtol(se+1, &tmp, 10) - 1;
- if ( tmp==se+1 ) error("Could not parse the region:%s\n",line);
-
- *chr_beg = ss;
- *chr_end = se-1;
-
- // skip region
- while ( *ss && !isspace(*ss) ) ss++;
- while ( *ss && isspace(*ss) ) ss++;
-
- rule_t *rule = (rule_t*) payload;
- memset(rule, 0, sizeof(rule_t));
-
- // mothernal ploidy
- se = ss;
- while ( *se && !isspace(*se) ) se++;
- int err = 0;
- if ( se - ss == 1 )
- {
- if ( *ss=='M' ) rule->mpl = 1;
- else if ( *ss=='.' ) rule->mpl = 0;
- else err = 1;
- }
- else if ( se - ss == 3 )
- {
- if ( !strncmp(ss,"M/M",3) ) rule->mpl = 2;
- else err = 1;
- }
- else err = 1;
- if ( err ) error("Could not parse the mothernal ploidy, only \"M\", \"M/M\" and \".\" currently supported: %s\n",line);
-
- // skip "+"
- while ( *se && isspace(*se) ) se++;
- if ( *se != '+' ) error("Could not parse the line: %s\n",line);
- se++;
- while ( *se && isspace(*se) ) se++;
-
- // paternal ploidy
- ss = se;
- while ( *se && !isspace(*se) ) se++;
- if ( se - ss == 1 )
- {
- if ( *ss=='F' ) rule->fpl = 1;
- else err = 1;
- }
- else err = 1;
- if ( err ) error("Could not parse the paternal ploidy, only \"F\" is currently supported: %s [%s]\n",line, ss);
-
- // skip ">"
- while ( *se && isspace(*se) ) se++;
- if ( *se != '>' ) error("Could not parse the line: %s\n",line);
- se++;
- while ( *se && isspace(*se) ) se++;
-
- // ploidy in offspring
- ss = se;
- while ( *se && !isspace(*se) ) se++;
- if ( se - ss == 3 )
- {
- if ( !strncmp(ss,"M/F",3) ) { rule->cpl = 2; rule->fal = 1; rule->mal = 1; }
- else err = 1;
- }
- else if ( se - ss == 1 )
- {
- if ( *ss=='F' ) { rule->cpl = 1; rule->fal = 1; }
- else if ( *ss=='M' ) { rule->cpl = 1; rule->mal = 1; }
- else err = 1;
- }
- else err = 1;
- if ( err ) error("Could not parse the offspring's ploidy, only \"M\", \"F\" or \"M/F\" is currently supported: %s\n",line);
-
- return 0;
-}
-
-int run(int argc, char **argv)
-{
- char *trio_samples = NULL, *trio_file = NULL, *rules_fname = NULL, *rules_string = NULL;
- memset(&args,0,sizeof(args_t));
- args.mode = 0;
- args.output_fname = "-";
-
- static struct option loptions[] =
- {
- {"trio",1,0,'t'},
- {"trio-file",1,0,'T'},
- {"delete",0,0,'d'},
- {"list",1,0,'l'},
- {"count",0,0,'c'},
- {"rules",1,0,'r'},
- {"rules-file",1,0,'R'},
- {"output",required_argument,NULL,'o'},
- {"output-type",required_argument,NULL,'O'},
- {0,0,0,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?ht:T:l:cdr:R:o:O:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'o': args.output_fname = optarg; break;
- case 'O':
- switch (optarg[0]) {
- case 'b': args.output_type = FT_BCF_GZ; break;
- case 'u': args.output_type = FT_BCF; break;
- case 'z': args.output_type = FT_VCF_GZ; break;
- case 'v': args.output_type = FT_VCF; break;
- default: error("The output type \"%s\" not recognised\n", optarg);
- };
- break;
- case 'R': rules_fname = optarg; break;
- case 'r': rules_string = optarg; break;
- case 'd': args.mode |= MODE_DELETE; break;
- case 'c': args.mode |= MODE_COUNT; break;
- case 'l':
- if ( !strcmp("+",optarg) ) args.mode |= MODE_LIST_GOOD;
- else if ( !strcmp("x",optarg) ) args.mode |= MODE_LIST_BAD;
- else error("The argument not recognised: --list %s\n", optarg);
- break;
- case 't': trio_samples = optarg; break;
- case 'T': trio_file = optarg; break;
- case 'h':
- case '?':
- default: error("%s",usage()); break;
- }
- }
- if ( rules_fname )
- args.rules = regidx_init(rules_fname, parse_rules, NULL, sizeof(rule_t), &args);
- else
- args.rules = init_rules(&args, rules_string);
- if ( !args.rules ) return -1;
- args.itr = regitr_init(args.rules);
- args.itr_ori = regitr_init(args.rules);
-
- char *fname = NULL;
- if ( optind>=argc || argv[optind][0]=='-' )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) fname = "-"; // reading from stdin
- else error("%s",usage());
- }
- else
- fname = argv[optind];
-
- if ( !trio_samples && !trio_file ) error("Expected the -t/T option\n");
- if ( !args.mode ) error("Expected one of the -c, -d or -l options\n");
- if ( args.mode&MODE_DELETE && !(args.mode&(MODE_LIST_GOOD|MODE_LIST_BAD)) ) args.mode |= MODE_LIST_GOOD|MODE_LIST_BAD;
-
- args.sr = bcf_sr_init();
- if ( !bcf_sr_add_reader(args.sr, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args.sr->errnum));
- args.hdr = bcf_sr_get_header(args.sr, 0);
- args.out_fh = hts_open(args.output_fname,hts_bcf_wmode(args.output_type));
- if ( args.out_fh == NULL ) error("Can't write to \"%s\": %s\n", args.output_fname, strerror(errno));
- bcf_hdr_write(args.out_fh, args.hdr);
-
-
- int i, n = 0;
- char **list;
- if ( trio_samples )
- {
- args.ntrios = 1;
- args.trios = (trio_t*) calloc(1,sizeof(trio_t));
- list = hts_readlist(trio_samples, 0, &n);
- if ( n!=3 ) error("Expected three sample names with -t\n");
- args.trios[0].imother = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[0]);
- args.trios[0].ifather = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[1]);
- args.trios[0].ichild = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, list[2]);
- for (i=0; i<n; i++) free(list[i]);
- free(list);
- }
- if ( trio_file )
- {
- list = hts_readlist(trio_file, 1, &n);
- args.ntrios = n;
- args.trios = (trio_t*) calloc(n,sizeof(trio_t));
- for (i=0; i<n; i++)
- {
- char *ss = list[i], *se;
- se = strchr(ss, ',');
- if ( !se ) error("Could not parse %s: %s\n",trio_file, ss);
- *se = 0;
- args.trios[i].imother = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, ss);
- if ( args.trios[i].imother<0 ) error("No such sample: \"%s\"\n", ss);
- ss = ++se;
- se = strchr(ss, ',');
- if ( !se ) error("Could not parse %s\n",trio_file);
- *se = 0;
- args.trios[i].ifather = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, ss);
- if ( args.trios[i].ifather<0 ) error("No such sample: \"%s\"\n", ss);
- ss = ++se;
- if ( *ss=='\0' ) error("Could not parse %s\n",trio_file);
- args.trios[i].ichild = bcf_hdr_id2int(args.hdr, BCF_DT_SAMPLE, ss);
- if ( args.trios[i].ichild<0 ) error("No such sample: \"%s\"\n", ss);
- free(list[i]);
- }
- free(list);
- }
-
- while ( bcf_sr_next_line(args.sr) )
- {
- bcf1_t *line = bcf_sr_get_line(args.sr,0);
- line = process(line);
- if ( line )
- {
- if ( line->errcode ) error("TODO: Unchecked error (%d), exiting\n",line->errcode);
- bcf_write1(args.out_fh, args.hdr, line);
- }
- }
-
-
- fprintf(bcftools_stderr,"# [1]nOK\t[2]nBad\t[3]nSkipped\t[4]Trio\n");
- for (i=0; i<args.ntrios; i++)
- {
- trio_t *trio = &args.trios[i];
- fprintf(bcftools_stderr,"%d\t%d\t%d\t%s,%s,%s\n",
- trio->nok,trio->nbad,args.nrec-(trio->nok+trio->nbad),
- bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, trio->imother),
- bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, trio->ifather),
- bcf_hdr_int2id(args.hdr, BCF_DT_SAMPLE, trio->ichild)
- );
- }
- free(args.gt_arr);
- free(args.trios);
- regitr_destroy(args.itr);
- regitr_destroy(args.itr_ori);
- regidx_destroy(args.rules);
- bcf_sr_destroy(args.sr);
- if ( hts_close(args.out_fh)!=0 ) error("Error: close failed\n");
- return 0;
-}
-
-static void warn_ploidy(bcf1_t *rec)
-{
- static int warned = 0;
- if ( warned ) return;
- fprintf(bcftools_stderr,"Incorrect ploidy at %s:%d, skipping the trio. (This warning is printed only once.)\n", bcf_seqname(args.hdr,rec),rec->pos+1);
- warned = 1;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- bcf1_t *dflt = args.mode&MODE_LIST_GOOD ? rec : NULL;
- args.nrec++;
-
- if ( rec->n_allele > 63 ) return dflt; // we use 64bit bitmask below
-
- int ngt = bcf_get_genotypes(args.hdr, rec, &args.gt_arr, &args.ngt_arr);
- if ( ngt<0 ) return dflt;
- if ( ngt!=2*bcf_hdr_nsamples(args.hdr) && ngt!=bcf_hdr_nsamples(args.hdr) ) return dflt;
- ngt /= bcf_hdr_nsamples(args.hdr);
-
- int itr_set = regidx_overlap(args.rules, bcf_seqname(args.hdr,rec),rec->pos,rec->pos, args.itr_ori);
-
- int i, has_bad = 0, needs_update = 0;
- for (i=0; i<args.ntrios; i++)
- {
- int32_t a,b,c,d,e,f;
- trio_t *trio = &args.trios[i];
-
- a = args.gt_arr[ngt*trio->imother];
- b = ngt==2 ? args.gt_arr[ngt*trio->imother+1] : bcf_int32_vector_end;
- c = args.gt_arr[ngt*trio->ifather];
- d = ngt==2 ? args.gt_arr[ngt*trio->ifather+1] : bcf_int32_vector_end;
- e = args.gt_arr[ngt*trio->ichild];
- f = ngt==2 ? args.gt_arr[ngt*trio->ichild+1] : bcf_int32_vector_end;
-
- // skip sites with missing data in child
- if ( bcf_gt_is_missing(e) || bcf_gt_is_missing(f) ) continue;
-
- uint64_t mother = 0, father = 0,child1,child2;
-
- int is_ok = 0;
- if ( !itr_set )
- {
- if ( f==bcf_int32_vector_end ) { warn_ploidy(rec); continue; }
-
- // All M,F,C genotypes are diploid. Missing data are considered consistent.
- child1 = 1<<bcf_gt_allele(e);
- child2 = 1<<bcf_gt_allele(f);
- mother = bcf_gt_is_missing(a) ? child1|child2 : 1<<bcf_gt_allele(a);
- mother |= bcf_gt_is_missing(b) || b==bcf_int32_vector_end ? child1|child2 : 1<<bcf_gt_allele(b);
- father = bcf_gt_is_missing(c) ? child1|child2 : 1<<bcf_gt_allele(c);
- father |= bcf_gt_is_missing(d) || d==bcf_int32_vector_end ? child1|child2 : 1<<bcf_gt_allele(d);
-
- if ( (mother&child1 && father&child2) || (mother&child2 && father&child1) ) is_ok = 1;
- }
- else
- {
- child1 = 1<<bcf_gt_allele(e);
- child2 = bcf_gt_is_missing(f) || f==bcf_int32_vector_end ? 0 : 1<<bcf_gt_allele(f);
- mother |= bcf_gt_is_missing(a) ? 0 : 1<<bcf_gt_allele(a);
- mother |= bcf_gt_is_missing(b) || b==bcf_int32_vector_end ? 0 : 1<<bcf_gt_allele(b);
- father |= bcf_gt_is_missing(c) ? 0 : 1<<bcf_gt_allele(c);
- father |= bcf_gt_is_missing(d) || d==bcf_int32_vector_end ? 0 : 1<<bcf_gt_allele(d);
-
- regitr_copy(args.itr, args.itr_ori);
- while ( !is_ok && regitr_overlap(args.itr) )
- {
- rule_t *rule = ®itr_payload(args.itr,rule_t);
- if ( child1 && child2 )
- {
- if ( !rule->mal || !rule->fal ) continue; // wrong rule (haploid), but this is a diploid GT
- if ( !mother ) mother = child1|child2;
- if ( !father ) father = child1|child2;
- if ( (mother&child1 && father&child2) || (mother&child2 && father&child1) ) is_ok = 1;
- continue;
- }
- if ( rule->mal )
- {
- if ( mother && !(child1&mother) ) continue;
- }
- if ( rule->fal )
- {
- if ( father && !(child1&father) ) continue;
- }
- is_ok = 1;
- }
- }
- if ( is_ok )
- {
- trio->nok++;
- }
- else
- {
- trio->nbad++;
- has_bad = 1;
- if ( args.mode&MODE_DELETE )
- {
- args.gt_arr[ngt*trio->imother] = bcf_gt_missing;
- if ( b!=bcf_int32_vector_end ) args.gt_arr[ngt*trio->imother+1] = bcf_gt_missing; // should be always true
- args.gt_arr[ngt*trio->ifather] = bcf_gt_missing;
- if ( d!=bcf_int32_vector_end ) args.gt_arr[ngt*trio->ifather+1] = bcf_gt_missing;
- args.gt_arr[ngt*trio->ichild] = bcf_gt_missing;
- if ( f!=bcf_int32_vector_end ) args.gt_arr[ngt*trio->ichild+1] = bcf_gt_missing;
- needs_update = 1;
- }
- }
- }
-
- if ( needs_update && bcf_update_genotypes(args.hdr,rec,args.gt_arr,ngt*bcf_hdr_nsamples(args.hdr)) )
- error("Could not update GT field at %s:%d\n", bcf_seqname(args.hdr,rec),rec->pos+1);
-
- if ( args.mode&MODE_DELETE ) return rec;
- if ( args.mode&MODE_LIST_GOOD ) return has_bad ? NULL : rec;
- if ( args.mode&MODE_LIST_BAD ) return has_bad ? rec : NULL;
-
- return NULL;
-}
-
+++ /dev/null
-/* plugins/missing2ref.c -- sets missing genotypes to reference allele.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-#include <htslib/vcfutils.h>
-#include <inttypes.h>
-#include <getopt.h>
-
-bcf_hdr_t *in_hdr, *out_hdr;
-int32_t *gts = NULL, mgts = 0;
-int *arr = NULL, marr = 0;
-uint64_t nchanged = 0;
-int new_gt = bcf_gt_unphased(0);
-int use_major = 0;
-
-const char *about(void)
-{
- return "Set missing genotypes (\"./.\") to ref or major allele (\"0/0\" or \"0|0\").\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Set missing genotypes\n"
- "Usage: bcftools +missing2ref [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -p, --phased Set to \"0|0\" \n"
- " -m, --major Set to major allele \n"
- "\n"
- "Example:\n"
- " bcftools +missing2ref in.vcf -- -p\n"
- " bcftools +missing2ref in.vcf -- -p -m\n"
- "\n";
-}
-
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- int c;
- static struct option loptions[] =
- {
- {"phased",0,0,'p'},
- {"major",0,0,'m'},
- {0,0,0,0}
- };
- while ((c = getopt_long(argc, argv, "mp?h",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'p': new_gt = bcf_gt_phased(0); break;
- case 'm': use_major = 1; break;
- case 'h':
- case '?':
- default: fprintf(stderr,"%s", usage()); exit(1); break;
- }
- }
- in_hdr = in;
- out_hdr = out;
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int ngts = bcf_get_genotypes(in_hdr, rec, >s, &mgts);
- int i, changed = 0;
-
- // Calculating allele frequency for each allele and determining major allele
- // only do this if use_major is true
- int majorAllele = -1;
- int maxAC = -1;
- int an = 0;
- if(use_major == 1){
- hts_expand(int,rec->n_allele,marr,arr);
- int ret = bcf_calc_ac(in_hdr,rec,arr,BCF_UN_FMT);
- if(ret > 0){
- for(i=0; i < rec->n_allele; ++i){
- an += arr[i];
- if(*(arr+i) > maxAC){
- maxAC = *(arr+i);
- majorAllele = i;
- }
- }
- }
- else{
- fprintf(stderr,"Warning: Could not calculate allele count at position %d\n", rec->pos);
- exit(1);
- }
-
- // replacing new_gt by major allele
- if(bcf_gt_is_phased(new_gt))
- new_gt = bcf_gt_phased(majorAllele);
- else
- new_gt = bcf_gt_unphased(majorAllele);
- }
-
- // replace gts
- for (i=0; i<ngts; i++)
- {
- if ( gts[i]==bcf_gt_missing )
- {
- gts[i] = new_gt;
- changed++;
- }
- }
- nchanged += changed;
- if ( changed ) bcf_update_genotypes(out_hdr, rec, gts, ngts);
- return rec;
-}
-
-void destroy(void)
-{
- free(arr);
- fprintf(stderr,"Filled %"PRId64" REF alleles\n", nchanged);
- free(gts);
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugins/missing2ref.c -- sets missing genotypes to reference allele.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-#include <htslib/vcfutils.h>
-#include <inttypes.h>
-#include <getopt.h>
-
-bcf_hdr_t *in_hdr, *out_hdr;
-int32_t *gts = NULL, mgts = 0;
-int *arr = NULL, marr = 0;
-uint64_t nchanged = 0;
-int new_gt = bcf_gt_unphased(0);
-int use_major = 0;
-
-const char *about(void)
-{
- return "Set missing genotypes (\"./.\") to ref or major allele (\"0/0\" or \"0|0\").\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Set missing genotypes\n"
- "Usage: bcftools +missing2ref [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -p, --phased Set to \"0|0\" \n"
- " -m, --major Set to major allele \n"
- "\n"
- "Example:\n"
- " bcftools +missing2ref in.vcf -- -p\n"
- " bcftools +missing2ref in.vcf -- -p -m\n"
- "\n";
-}
-
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- int c;
- static struct option loptions[] =
- {
- {"phased",0,0,'p'},
- {"major",0,0,'m'},
- {0,0,0,0}
- };
- while ((c = getopt_long(argc, argv, "mp?h",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'p': new_gt = bcf_gt_phased(0); break;
- case 'm': use_major = 1; break;
- case 'h':
- case '?':
- default: fprintf(bcftools_stderr,"%s", usage()); exit(1); break;
- }
- }
- in_hdr = in;
- out_hdr = out;
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int ngts = bcf_get_genotypes(in_hdr, rec, >s, &mgts);
- int i, changed = 0;
-
- // Calculating allele frequency for each allele and determining major allele
- // only do this if use_major is true
- int majorAllele = -1;
- int maxAC = -1;
- int an = 0;
- if(use_major == 1){
- hts_expand(int,rec->n_allele,marr,arr);
- int ret = bcf_calc_ac(in_hdr,rec,arr,BCF_UN_FMT);
- if(ret > 0){
- for(i=0; i < rec->n_allele; ++i){
- an += arr[i];
- if(*(arr+i) > maxAC){
- maxAC = *(arr+i);
- majorAllele = i;
- }
- }
- }
- else{
- fprintf(bcftools_stderr,"Warning: Could not calculate allele count at position %d\n", rec->pos);
- exit(1);
- }
-
- // replacing new_gt by major allele
- if(bcf_gt_is_phased(new_gt))
- new_gt = bcf_gt_phased(majorAllele);
- else
- new_gt = bcf_gt_unphased(majorAllele);
- }
-
- // replace gts
- for (i=0; i<ngts; i++)
- {
- if ( gts[i]==bcf_gt_missing )
- {
- gts[i] = new_gt;
- changed++;
- }
- }
- nchanged += changed;
- if ( changed ) bcf_update_genotypes(out_hdr, rec, gts, ngts);
- return rec;
-}
-
-void destroy(void)
-{
- free(arr);
- fprintf(bcftools_stderr,"Filled %"PRId64" REF alleles\n", nchanged);
- free(gts);
-}
-
-
+++ /dev/null
-/*
- Copyright (C) 2017 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-/*
- Prune sites by missingness, LD
-
- See calc_ld() in vcfbuf.c for the actual LD calculation
-
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <unistd.h>
-#include <stdint.h>
-#include <errno.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include "bcftools.h"
-#include "vcfbuf.h"
-#include "filter.h"
-
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-typedef struct
-{
- filter_t *filter;
- char *filter_str, *af_tag;
- int filter_logic; // one of FLT_INCLUDE/FLT_EXCLUDE (-i or -e)
- vcfbuf_t *vcfbuf;
- int argc, region_is_file, target_is_file, output_type, filter_r2_id, rand_missing, nsites, ld_win;
- char **argv, *region, *target, *fname, *output_fname, *info_pos, *info_r2, *filter_r2;
- htsFile *out_fh;
- bcf_hdr_t *hdr;
- bcf_srs_t *sr;
- double max_ld;
-}
-args_t;
-
-const char *about(void)
-{
- return "Prune sites by missingness, linkage disequilibrium\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Prune sites by missingness or linkage disequilibrium.\n"
- "\n"
- "Usage: bcftools +prune [Options]\n"
- "Plugin options:\n"
- " --AF-tag STR use this tag with -n to determine allele frequency\n"
- " -a, --annotate-info STR add INFO/STR_POS and STR_R2 annotation: an upstream site with the biggest r2 value\n"
- " -e, --exclude EXPR exclude sites for which the expression is true\n"
- " -f, --set-filter STR annotate FILTER column with STR instead of discarding the site\n"
- " -i, --include EXPR include only sites for which the expression is true\n"
- " -l, --max-LD R2 remove sites with r2 bigger than R2 within within the -w window\n"
- " -n, --nsites-per-win N keep at most N sites in the -w window, removing sites with small AF first\n"
- " -o, --output FILE write output to the FILE [standard output]\n"
- " -O, --output-type b|u|z|v b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n"
- " --randomize-missing replace missing data with randomly assigned genotype based on site's allele frequency\n"
- " -r, --regions REGION restrict to comma-separated list of regions\n"
- " -R, --regions-file FILE restrict to regions listed in a file\n"
- " -t, --targets REGION similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file FILE similar to -R but streams rather than index-jumps\n"
- " -w, --window INT[bp|kb] the window size of INT sites/bp/kb for the -n/-l options [100kb]\n"
- "Examples:\n"
- " # Discard records with r2 bigger than 0.6 in a window of 1000 sites\n"
- " bcftools +prune -l 0.6 -w 1000 input.bcf -Ob -o output.bcf\n"
- "\n"
- " # Set FILTER (but do not discard) records with r2 bigger than 0.4 in the default window of 100kb\n"
- " bcftools +prune -l 0.4 -f MAX_R2 input.bcf -Ob -o output.bcf\n"
- "\n"
- " # Annotate INFO field of all records with maximum r2 in a window of 1000 sites\n"
- " bcftools +prune -l 0.6 -w 1000 -f MAX_R2 input.bcf -Ob -o output.bcf\n"
- "\n"
- " # Discard records with r2 bigger than 0.6, first removing records with more than 2% of genotypes missing\n"
- " bcftools +prune -l 0.6 -e'F_MISSING>=0.02' input.bcf -Ob -o output.bcf\n"
- "\n";
-}
-
-static void init_data(args_t *args)
-{
- args->sr = bcf_sr_init();
- if ( args->region )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, args->region, args->region_is_file)<0 ) error("Failed to read the regions: %s\n",args->region);
- }
- if ( args->target && bcf_sr_set_targets(args->sr, args->target, args->target_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->target);
- if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr = bcf_sr_get_header(args->sr,0);
-
- args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
- if ( args->filter_r2 )
- {
- bcf_hdr_printf(args->hdr,"##FILTER=<ID=%s,Description=\"A site with r2>%e upstream within %d%s\">",args->filter_r2,args->max_ld,
- args->ld_win < 0 ? -args->ld_win/1000 : args->ld_win,
- args->ld_win < 0 ? "kb" : " sites");
- }
- if ( args->info_r2 )
- {
- bcf_hdr_printf(args->hdr,"##INFO=<ID=%s,Number=1,Type=Integer,Description=\"A site with r2>%e upstream\">",args->info_pos,args->max_ld);
- bcf_hdr_printf(args->hdr,"##INFO=<ID=%s,Number=1,Type=Float,Description=\"A site with r2>%e upstream\">",args->info_r2,args->max_ld);
- }
- bcf_hdr_write(args->out_fh, args->hdr);
- if ( args->filter_r2 )
- args->filter_r2_id = bcf_hdr_id2int(args->hdr, BCF_DT_ID, args->filter_r2);
-
- args->vcfbuf = vcfbuf_init(args->hdr, args->ld_win);
- vcfbuf_set_opt(args->vcfbuf,double,VCFBUF_LD_MAX,args->max_ld);
- if ( args->nsites ) vcfbuf_set_opt(args->vcfbuf,int,VCFBUF_NSITES,args->nsites);
- if ( args->af_tag ) vcfbuf_set_opt(args->vcfbuf,char*,VCFBUF_AF_TAG,args->af_tag);
- if ( args->rand_missing ) vcfbuf_set_opt(args->vcfbuf,int,VCFBUF_RAND_MISSING,1);
- vcfbuf_set_opt(args->vcfbuf,int,VCFBUF_SKIP_FILTER,args->filter_r2 ? 1 : 0);
-
- if ( args->filter_str )
- args->filter = filter_init(args->hdr, args->filter_str);
-}
-static void destroy_data(args_t *args)
-{
- if ( args->filter )
- filter_destroy(args->filter);
- hts_close(args->out_fh);
- vcfbuf_destroy(args->vcfbuf);
- bcf_sr_destroy(args->sr);
- free(args->info_pos);
- free(args->info_r2);
- free(args);
-}
-static void flush(args_t *args, int flush_all)
-{
- bcf1_t *rec;
- while ( (rec = vcfbuf_flush(args->vcfbuf, flush_all)) )
- bcf_write1(args->out_fh, args->hdr, rec);
-}
-static void process(args_t *args)
-{
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- if ( args->filter )
- {
- int ret = filter_test(args->filter, rec, NULL);
- if ( args->filter_logic==FLT_INCLUDE ) { if ( !ret ) return; }
- else if ( ret ) return;
- }
- bcf_sr_t *sr = bcf_sr_get_reader(args->sr, 0);
- if ( args->max_ld )
- {
- double ld_val;
- bcf1_t *ld_rec = vcfbuf_max_ld(args->vcfbuf, rec, &ld_val);
- if ( ld_rec && ld_val > args->max_ld )
- {
- if ( !args->filter_r2 ) return;
- bcf_add_filter(args->hdr, rec, args->filter_r2_id);
- }
- if ( ld_rec && args->info_r2 )
- {
- float tmp = ld_val;
- int32_t tmp_pos = ld_rec->pos + 1;
- bcf_update_info_float(args->hdr, rec, args->info_r2, &tmp, 1);
- bcf_update_info_int32(args->hdr, rec, args->info_pos, &tmp_pos, 1);
- }
- }
- sr->buffer[0] = vcfbuf_push(args->vcfbuf, rec, 1);
- flush(args,0);
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->output_type = FT_VCF;
- args->output_fname = "-";
- args->ld_win = -100e3;
- static struct option loptions[] =
- {
- {"randomize-missing",no_argument,NULL,1},
- {"AF-tag",required_argument,NULL,2},
- {"exclude",required_argument,NULL,'e'},
- {"include",required_argument,NULL,'i'},
- {"annotate-info",required_argument,NULL,'a'},
- {"set-filter",required_argument,NULL,'f'},
- {"max-LD",required_argument,NULL,'l'},
- {"regions",required_argument,NULL,'r'},
- {"regions-file",required_argument,NULL,'R'},
- {"output",required_argument,NULL,'o'},
- {"output-type",required_argument,NULL,'O'},
- {"nsites-per-win",required_argument,NULL,'n'},
- {"window",required_argument,NULL,'w'},
- {NULL,0,NULL,0}
- };
- int c;
- char *tmp;
- while ((c = getopt_long(argc, argv, "vr:R:t:T:l:o:O:a:f:i:e:n:w:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 1 : args->rand_missing = 1; break;
- case 2 : args->af_tag = optarg; break;
- case 'e': args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 'a':
- {
- int l = strlen(optarg);
- args->info_pos = (char*)malloc(l+5);
- args->info_r2 = (char*)malloc(l+5);
- sprintf(args->info_pos,"%s_POS", optarg);
- sprintf(args->info_r2,"%s_R2", optarg);
- }
- break;
- case 'f': args->filter_r2 = optarg; break;
- case 'n':
- args->nsites = strtod(optarg,&tmp);
- if ( tmp==optarg || *tmp ) error("Could not parse: --nsites-per-win %s\n", optarg);
- break;
- case 'l':
- args->max_ld = strtod(optarg,&tmp);
- if ( tmp==optarg || *tmp ) error("Could not parse: --max-LD %s\n", optarg);
- break;
- case 'w':
- args->ld_win = strtod(optarg,&tmp);
- if ( !*tmp ) break;
- if ( tmp==optarg ) error("Could not parse: --window %s\n", optarg);
- else if ( !strcasecmp("bp",tmp) ) args->ld_win *= -1;
- else if ( !strcasecmp("kb",tmp) ) args->ld_win *= -1000;
- else error("Could not parse: --window %s\n", optarg);
- break;
- case 'T': args->target_is_file = 1;
- case 't': args->target = optarg; break;
- case 'R': args->region_is_file = 1;
- case 'r': args->region = optarg; break;
- case 'o': args->output_fname = optarg; break;
- case 'O':
- switch (optarg[0]) {
- case 'b': args->output_type = FT_BCF_GZ; break;
- case 'u': args->output_type = FT_BCF; break;
- case 'z': args->output_type = FT_VCF_GZ; break;
- case 'v': args->output_type = FT_VCF; break;
- default: error("The output type \"%s\" not recognised\n", optarg);
- }
- break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( args->filter_logic == (FLT_EXCLUDE|FLT_INCLUDE) ) error("Only one of -i or -e can be given.\n");
- if ( !args->max_ld && !args->nsites ) error("%sError: Expected --max-LD, --nsites-per-win or both\n\n", usage_text());
-
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else { error("%s",usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s",usage_text());
- else args->fname = argv[optind];
-
- init_data(args);
-
- while ( bcf_sr_next_line(args->sr) ) process(args);
- flush(args,1);
-
- destroy_data(args);
- return 0;
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/*
- Copyright (C) 2017 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-/*
- Prune sites by missingness, LD
-
- See calc_ld() in vcfbuf.c for the actual LD calculation
-
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <unistd.h>
-#include <stdint.h>
-#include <errno.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include "bcftools.h"
-#include "vcfbuf.h"
-#include "filter.h"
-
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-typedef struct
-{
- filter_t *filter;
- char *filter_str, *af_tag;
- int filter_logic; // one of FLT_INCLUDE/FLT_EXCLUDE (-i or -e)
- vcfbuf_t *vcfbuf;
- int argc, region_is_file, target_is_file, output_type, filter_r2_id, rand_missing, nsites, ld_win;
- char **argv, *region, *target, *fname, *output_fname, *info_pos, *info_r2, *filter_r2;
- htsFile *out_fh;
- bcf_hdr_t *hdr;
- bcf_srs_t *sr;
- double max_ld;
-}
-args_t;
-
-const char *about(void)
-{
- return "Prune sites by missingness, linkage disequilibrium\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Prune sites by missingness or linkage disequilibrium.\n"
- "\n"
- "Usage: bcftools +prune [Options]\n"
- "Plugin options:\n"
- " --AF-tag STR use this tag with -n to determine allele frequency\n"
- " -a, --annotate-info STR add INFO/STR_POS and STR_R2 annotation: an upstream site with the biggest r2 value\n"
- " -e, --exclude EXPR exclude sites for which the expression is true\n"
- " -f, --set-filter STR annotate FILTER column with STR instead of discarding the site\n"
- " -i, --include EXPR include only sites for which the expression is true\n"
- " -l, --max-LD R2 remove sites with r2 bigger than R2 within within the -w window\n"
- " -n, --nsites-per-win N keep at most N sites in the -w window, removing sites with small AF first\n"
- " -o, --output FILE write output to the FILE [standard output]\n"
- " -O, --output-type b|u|z|v b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n"
- " --randomize-missing replace missing data with randomly assigned genotype based on site's allele frequency\n"
- " -r, --regions REGION restrict to comma-separated list of regions\n"
- " -R, --regions-file FILE restrict to regions listed in a file\n"
- " -t, --targets REGION similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file FILE similar to -R but streams rather than index-jumps\n"
- " -w, --window INT[bp|kb] the window size of INT sites/bp/kb for the -n/-l options [100kb]\n"
- "Examples:\n"
- " # Discard records with r2 bigger than 0.6 in a window of 1000 sites\n"
- " bcftools +prune -l 0.6 -w 1000 input.bcf -Ob -o output.bcf\n"
- "\n"
- " # Set FILTER (but do not discard) records with r2 bigger than 0.4 in the default window of 100kb\n"
- " bcftools +prune -l 0.4 -f MAX_R2 input.bcf -Ob -o output.bcf\n"
- "\n"
- " # Annotate INFO field of all records with maximum r2 in a window of 1000 sites\n"
- " bcftools +prune -l 0.6 -w 1000 -f MAX_R2 input.bcf -Ob -o output.bcf\n"
- "\n"
- " # Discard records with r2 bigger than 0.6, first removing records with more than 2% of genotypes missing\n"
- " bcftools +prune -l 0.6 -e'F_MISSING>=0.02' input.bcf -Ob -o output.bcf\n"
- "\n";
-}
-
-static void init_data(args_t *args)
-{
- args->sr = bcf_sr_init();
- if ( args->region )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, args->region, args->region_is_file)<0 ) error("Failed to read the regions: %s\n",args->region);
- }
- if ( args->target && bcf_sr_set_targets(args->sr, args->target, args->target_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->target);
- if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr = bcf_sr_get_header(args->sr,0);
-
- args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
- if ( args->filter_r2 )
- {
- bcf_hdr_printf(args->hdr,"##FILTER=<ID=%s,Description=\"A site with r2>%e upstream within %d%s\">",args->filter_r2,args->max_ld,
- args->ld_win < 0 ? -args->ld_win/1000 : args->ld_win,
- args->ld_win < 0 ? "kb" : " sites");
- }
- if ( args->info_r2 )
- {
- bcf_hdr_printf(args->hdr,"##INFO=<ID=%s,Number=1,Type=Integer,Description=\"A site with r2>%e upstream\">",args->info_pos,args->max_ld);
- bcf_hdr_printf(args->hdr,"##INFO=<ID=%s,Number=1,Type=Float,Description=\"A site with r2>%e upstream\">",args->info_r2,args->max_ld);
- }
- bcf_hdr_write(args->out_fh, args->hdr);
- if ( args->filter_r2 )
- args->filter_r2_id = bcf_hdr_id2int(args->hdr, BCF_DT_ID, args->filter_r2);
-
- args->vcfbuf = vcfbuf_init(args->hdr, args->ld_win);
- vcfbuf_set_opt(args->vcfbuf,double,VCFBUF_LD_MAX,args->max_ld);
- if ( args->nsites ) vcfbuf_set_opt(args->vcfbuf,int,VCFBUF_NSITES,args->nsites);
- if ( args->af_tag ) vcfbuf_set_opt(args->vcfbuf,char*,VCFBUF_AF_TAG,args->af_tag);
- if ( args->rand_missing ) vcfbuf_set_opt(args->vcfbuf,int,VCFBUF_RAND_MISSING,1);
- vcfbuf_set_opt(args->vcfbuf,int,VCFBUF_SKIP_FILTER,args->filter_r2 ? 1 : 0);
-
- if ( args->filter_str )
- args->filter = filter_init(args->hdr, args->filter_str);
-}
-static void destroy_data(args_t *args)
-{
- if ( args->filter )
- filter_destroy(args->filter);
- hts_close(args->out_fh);
- vcfbuf_destroy(args->vcfbuf);
- bcf_sr_destroy(args->sr);
- free(args->info_pos);
- free(args->info_r2);
- free(args);
-}
-static void flush(args_t *args, int flush_all)
-{
- bcf1_t *rec;
- while ( (rec = vcfbuf_flush(args->vcfbuf, flush_all)) )
- bcf_write1(args->out_fh, args->hdr, rec);
-}
-static void process(args_t *args)
-{
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- if ( args->filter )
- {
- int ret = filter_test(args->filter, rec, NULL);
- if ( args->filter_logic==FLT_INCLUDE ) { if ( !ret ) return; }
- else if ( ret ) return;
- }
- bcf_sr_t *sr = bcf_sr_get_reader(args->sr, 0);
- if ( args->max_ld )
- {
- double ld_val;
- bcf1_t *ld_rec = vcfbuf_max_ld(args->vcfbuf, rec, &ld_val);
- if ( ld_rec && ld_val > args->max_ld )
- {
- if ( !args->filter_r2 ) return;
- bcf_add_filter(args->hdr, rec, args->filter_r2_id);
- }
- if ( ld_rec && args->info_r2 )
- {
- float tmp = ld_val;
- int32_t tmp_pos = ld_rec->pos + 1;
- bcf_update_info_float(args->hdr, rec, args->info_r2, &tmp, 1);
- bcf_update_info_int32(args->hdr, rec, args->info_pos, &tmp_pos, 1);
- }
- }
- sr->buffer[0] = vcfbuf_push(args->vcfbuf, rec, 1);
- flush(args,0);
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->output_type = FT_VCF;
- args->output_fname = "-";
- args->ld_win = -100e3;
- static struct option loptions[] =
- {
- {"randomize-missing",no_argument,NULL,1},
- {"AF-tag",required_argument,NULL,2},
- {"exclude",required_argument,NULL,'e'},
- {"include",required_argument,NULL,'i'},
- {"annotate-info",required_argument,NULL,'a'},
- {"set-filter",required_argument,NULL,'f'},
- {"max-LD",required_argument,NULL,'l'},
- {"regions",required_argument,NULL,'r'},
- {"regions-file",required_argument,NULL,'R'},
- {"output",required_argument,NULL,'o'},
- {"output-type",required_argument,NULL,'O'},
- {"nsites-per-win",required_argument,NULL,'n'},
- {"window",required_argument,NULL,'w'},
- {NULL,0,NULL,0}
- };
- int c;
- char *tmp;
- while ((c = getopt_long(argc, argv, "vr:R:t:T:l:o:O:a:f:i:e:n:w:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 1 : args->rand_missing = 1; break;
- case 2 : args->af_tag = optarg; break;
- case 'e': args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 'a':
- {
- int l = strlen(optarg);
- args->info_pos = (char*)malloc(l+5);
- args->info_r2 = (char*)malloc(l+5);
- sprintf(args->info_pos,"%s_POS", optarg);
- sprintf(args->info_r2,"%s_R2", optarg);
- }
- break;
- case 'f': args->filter_r2 = optarg; break;
- case 'n':
- args->nsites = strtod(optarg,&tmp);
- if ( tmp==optarg || *tmp ) error("Could not parse: --nsites-per-win %s\n", optarg);
- break;
- case 'l':
- args->max_ld = strtod(optarg,&tmp);
- if ( tmp==optarg || *tmp ) error("Could not parse: --max-LD %s\n", optarg);
- break;
- case 'w':
- args->ld_win = strtod(optarg,&tmp);
- if ( !*tmp ) break;
- if ( tmp==optarg ) error("Could not parse: --window %s\n", optarg);
- else if ( !strcasecmp("bp",tmp) ) args->ld_win *= -1;
- else if ( !strcasecmp("kb",tmp) ) args->ld_win *= -1000;
- else error("Could not parse: --window %s\n", optarg);
- break;
- case 'T': args->target_is_file = 1;
- case 't': args->target = optarg; break;
- case 'R': args->region_is_file = 1;
- case 'r': args->region = optarg; break;
- case 'o': args->output_fname = optarg; break;
- case 'O':
- switch (optarg[0]) {
- case 'b': args->output_type = FT_BCF_GZ; break;
- case 'u': args->output_type = FT_BCF; break;
- case 'z': args->output_type = FT_VCF_GZ; break;
- case 'v': args->output_type = FT_VCF; break;
- default: error("The output type \"%s\" not recognised\n", optarg);
- }
- break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( args->filter_logic == (FLT_EXCLUDE|FLT_INCLUDE) ) error("Only one of -i or -e can be given.\n");
- if ( !args->max_ld && !args->nsites ) error("%sError: Expected --max-LD, --nsites-per-win or both\n\n", usage_text());
-
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else { error("%s",usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s",usage_text());
- else args->fname = argv[optind];
-
- init_data(args);
-
- while ( bcf_sr_next_line(args->sr) ) process(args);
- flush(args,1);
-
- destroy_data(args);
- return 0;
-}
-
-
+++ /dev/null
-/* plugins/setGT.c -- set gentoypes to given values
-
- Copyright (C) 2015-2017 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-#include <htslib/vcfutils.h>
-#include <htslib/kfunc.h>
-#include <inttypes.h>
-#include <getopt.h>
-#include <ctype.h>
-#include "bcftools.h"
-#include "filter.h"
-
-// Logic of the filters: include or exclude sites which match the filters?
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-typedef int (*cmp_f)(double a, double b);
-
-static int cmp_eq(double a, double b) { return a==b ? 1 : 0; }
-static int cmp_le(double a, double b) { return a<=b ? 1 : 0; }
-static int cmp_ge(double a, double b) { return a>=b ? 1 : 0; }
-static int cmp_lt(double a, double b) { return a<b ? 1 : 0; }
-static int cmp_gt(double a, double b) { return a>b ? 1 : 0; }
-
-typedef struct
-{
- bcf_hdr_t *in_hdr, *out_hdr;
- int32_t *gts, mgts, *iarr, miarr;
- int *arr, marr;
- uint64_t nchanged;
- int tgt_mask, new_mask, new_gt;
- filter_t *filter;
- char *filter_str;
- int filter_logic;
- uint8_t *smpl_pass;
- double binom_val;
- char *binom_tag;
- cmp_f binom_cmp;
-}
-args_t;
-
-args_t *args = NULL;
-
-#define GT_MISSING 1
-#define GT_PARTIAL (1<<1)
-#define GT_REF (1<<2)
-#define GT_MAJOR (1<<3)
-#define GT_PHASED (1<<4)
-#define GT_UNPHASED (1<<5)
-#define GT_ALL (1<<6)
-#define GT_QUERY (1<<7)
-#define GT_BINOM (1<<8)
-
-const char *about(void)
-{
- return "Set genotypes: partially missing to missing, missing to ref/major allele, etc.\n";
-}
-
-const char *usage(void)
-{
- return
- "About: Sets genotypes. The target genotypes can be specified as:\n"
- " ./. .. completely missing (\".\" or \"./.\", depending on ploidy)\n"
- " ./x .. partially missing (e.g., \"./0\" or \".|1\" but not \"./.\")\n"
- " . .. partially or completely missing\n"
- " a .. all genotypes\n"
- " b .. heterozygous genotypes failing two-tailed binomial test (example below)\n"
- " q .. select genotypes using -i/-e options\n"
- " and the new genotype can be one of:\n"
- " . .. missing (\".\" or \"./.\", keeps ploidy)\n"
- " 0 .. reference allele\n"
- " M .. major allele\n"
- " p .. phased genotype\n"
- " u .. unphase genotype and sort by allele (1|0 becomes 0/1)\n"
- "Usage: bcftools +setGT [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -e, --exclude <expr> Exclude a genotype if true (requires -t q)\n"
- " -i, --include <expr> include a genotype if true (requires -t q)\n"
- " -n, --new-gt <type> Genotypes to set, see above\n"
- " -t, --target-gt <type> Genotypes to change, see above\n"
- "\n"
- "Example:\n"
- " # set missing genotypes (\"./.\") to phased ref genotypes (\"0|0\")\n"
- " bcftools +setGT in.vcf -- -t . -n 0p\n"
- "\n"
- " # set missing genotypes with DP>0 and GQ>20 to ref genotypes (\"0/0\")\n"
- " bcftools +setGT in.vcf -- -t q -n 0 -i 'GT=\".\" && FMT/DP>0 && GQ>20'\n"
- "\n"
- " # set partially missing genotypes to completely missing\n"
- " bcftools +setGT in.vcf -- -t ./x -n .\n"
- "\n"
- " # set heterozygous genotypes to 0/0 if binom.test(nAlt,nRef+nAlt,0.5)<1e-3\n"
- " bcftools +setGT in.vcf -- -t \"b:AD<1e-3\" -n 0\n" // todo: make -i/-e recognise something like is_het or gt="het" so that this can be generalized?
- "\n";
-}
-
-static void _parse_binom_expr_error(char *str)
-{
- error(
- "Error parsing the expression: %s\n"
- "Expected TAG CMP VAL, where\n"
- " TAG .. one of the format tags\n"
- " CMP .. operator, one of <, <=, >, >=\n"
- " VAL .. value\n"
- "For example:\n"
- " bcftools +setGT in.vcf -- -t \"b:AD>1e-3\" -n 0\n"
- "\n", str
- );
-}
-void parse_binom_expr(args_t *args, char *str)
-{
- if ( str[1]!=':' ) _parse_binom_expr_error(str);
-
- char *beg = str+2;
- while ( *beg && isspace(*beg) ) beg++;
- if ( !*beg ) _parse_binom_expr_error(str);
- char *end = beg;
- while ( *end )
- {
- if ( isspace(*end) || *end=='<' || *end=='=' || *end=='>' ) break;
- end++;
- }
- if ( !*end ) _parse_binom_expr_error(str);
- args->binom_tag = (char*) calloc(1,end-beg+1);
- memcpy(args->binom_tag,beg,end-beg);
- int tag_id = bcf_hdr_id2int(args->in_hdr,BCF_DT_ID,args->binom_tag);
- if ( !bcf_hdr_idinfo_exists(args->in_hdr,BCF_HL_FMT,tag_id) ) error("The FORMAT tag \"%s\" is not present in the VCF\n", args->binom_tag);
-
- while ( *end && isspace(*end) ) end++;
- if ( !*end ) _parse_binom_expr_error(str);
-
- if ( !strncmp(end,"<=",2) ) { args->binom_cmp = cmp_le; beg = end+2; }
- else if ( !strncmp(end,">=",2) ) { args->binom_cmp = cmp_ge; beg = end+2; }
- else if ( !strncmp(end,"==",2) ) { args->binom_cmp = cmp_eq; beg = end+2; }
- else if ( !strncmp(end,"<",1) ) { args->binom_cmp = cmp_lt; beg = end+1; }
- else if ( !strncmp(end,">",1) ) { args->binom_cmp = cmp_gt; beg = end+1; }
- else if ( !strncmp(end,"=",1) ) { args->binom_cmp = cmp_eq; beg = end+1; }
- else _parse_binom_expr_error(str);
-
- while ( *beg && isspace(*beg) ) beg++;
- if ( !*beg ) _parse_binom_expr_error(str);
-
- args->binom_val = strtod(beg, &end);
- while ( *end && isspace(*end) ) end++;
- if ( *end ) _parse_binom_expr_error(str);
-
- args->tgt_mask |= GT_BINOM;
- return;
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- args = (args_t*) calloc(1,sizeof(args_t));
- args->in_hdr = in;
- args->out_hdr = out;
-
- int c;
- static struct option loptions[] =
- {
- {"include",required_argument,NULL,'i'},
- {"exclude",required_argument,NULL,'e'},
- {"new-gt",required_argument,NULL,'n'},
- {"target-gt",required_argument,NULL,'t'},
- {NULL,0,NULL,0}
- };
- while ((c = getopt_long(argc, argv, "?hn:t:i:e:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'i': args->filter_str = optarg; args->filter_logic = FLT_INCLUDE; break;
- case 'e': args->filter_str = optarg; args->filter_logic = FLT_EXCLUDE; break;
- case 'n': args->new_mask = 0;
- if ( strchr(optarg,'.') ) args->new_mask |= GT_MISSING;
- if ( strchr(optarg,'0') ) args->new_mask |= GT_REF;
- if ( strchr(optarg,'M') ) args->new_mask |= GT_MAJOR;
- if ( strchr(optarg,'p') ) args->new_mask |= GT_PHASED;
- if ( strchr(optarg,'u') ) args->new_mask |= GT_UNPHASED;
- if ( args->new_mask==0 ) error("Unknown parameter to --new-gt: %s\n", optarg);
- break;
- case 't':
- if ( !strcmp(optarg,".") ) args->tgt_mask |= GT_MISSING|GT_PARTIAL;
- if ( !strcmp(optarg,"./x") ) args->tgt_mask |= GT_PARTIAL;
- if ( !strcmp(optarg,"./.") ) args->tgt_mask |= GT_MISSING;
- if ( !strcmp(optarg,"a") ) args->tgt_mask |= GT_ALL;
- if ( !strcmp(optarg,"q") ) args->tgt_mask |= GT_QUERY;
- if ( !strcmp(optarg,"?") ) args->tgt_mask |= GT_QUERY; // for backward compatibility
- if ( strchr(optarg,'b') ) parse_binom_expr(args, strchr(optarg,'b'));
- if ( args->tgt_mask==0 ) error("Unknown parameter to --target-gt: %s\n", optarg);
- break;
- case 'h':
- case '?':
- default: fprintf(stderr,"%s", usage()); exit(1); break;
- }
- }
-
- if ( !args->new_mask ) error("Expected -n option\n");
- if ( !args->tgt_mask ) error("Expected -t option\n");
-
- if ( args->new_mask & GT_MISSING ) args->new_gt = bcf_gt_missing;
- if ( args->new_mask & GT_REF ) args->new_gt = args->new_mask>_PHASED ? bcf_gt_phased(0) : bcf_gt_unphased(0);
-
- if ( args->filter_str && !(args->tgt_mask>_QUERY) ) error("Expected -tq with -i/-e\n");
- if ( !args->filter_str && args->tgt_mask>_QUERY ) error("Expected -i/-e with -tq\n");
- if ( args->filter_str ) args->filter = filter_init(in,args->filter_str);
-
- return 0;
-}
-
-static inline int phase_gt(int32_t *ptr, int ngts)
-{
- int j, changed = 0;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( bcf_gt_is_phased(ptr[j]) ) continue;
- ptr[j] = bcf_gt_phased(bcf_gt_allele(ptr[j])); // add phasing; this may need a fix, I think the flag should be set for one allele only?
- changed++;
- }
- return changed;
-}
-
-static inline int unphase_gt(int32_t *ptr, int ngts)
-{
- int j, changed = 0;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( !bcf_gt_is_phased(ptr[j]) ) continue;
- ptr[j] = bcf_gt_unphased(bcf_gt_allele(ptr[j])); // remove phasing
- changed++;
- }
-
- // insertion sort
- int k, l;
- for (k=1; k<j; k++)
- {
- int32_t x = ptr[k];
- l = k;
- while ( l>0 && ptr[l-1]>x )
- {
- ptr[l] = ptr[l-1];
- l--;
- }
- ptr[l] = x;
- }
- return changed;
-}
-static inline int set_gt(int32_t *ptr, int ngts, int gt)
-{
- int j, changed = 0;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( ptr[j] != gt ) changed++;
- ptr[j] = gt;
- }
- return changed;
-}
-
-static inline double calc_binom(int na, int nb)
-{
- if ( na + nb == 0 ) return 1;
-
- /*
- kfunc.h implements kf_betai, which is the regularized beta function I_x(a,b) = P(X<=a/(a+b))
- */
- double prob = na > nb ? 2*kf_betai(na, nb + 1, 0.5) : 2*kf_betai(nb, na + 1, 0.5);
- if ( prob > 1 ) prob = 1;
-
- return prob;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- if ( !rec->n_sample ) return rec;
-
- int ngts = bcf_get_genotypes(args->in_hdr, rec, &args->gts, &args->mgts);
- ngts /= rec->n_sample;
- int i, j, changed = 0;
-
- int nbinom = 0;
- if ( args->tgt_mask & GT_BINOM )
- {
- nbinom = bcf_get_format_int32(args->in_hdr, rec, args->binom_tag, &args->iarr, &args->miarr);
- if ( nbinom<0 ) nbinom = 0;
- nbinom /= rec->n_sample;
- }
-
- // Calculating allele frequency for each allele and determining major allele
- // only do this if use_major is true
- int an = 0, maxAC = -1, majorAllele = -1;
- if ( args->new_mask & GT_MAJOR )
- {
- hts_expand(int,rec->n_allele,args->marr,args->arr);
- int ret = bcf_calc_ac(args->in_hdr,rec,args->arr,BCF_UN_FMT);
- if ( ret<= 0 )
- error("Could not calculate allele count at %s:%d\n", bcf_seqname(args->in_hdr,rec),rec->pos+1);
-
- for(i=0; i < rec->n_allele; ++i)
- {
- an += args->arr[i];
- if (args->arr[i] > maxAC)
- {
- maxAC = args->arr[i];
- majorAllele = i;
- }
- }
-
- // replacing new_gt by major allele
- args->new_gt = args->new_mask & GT_PHASED ? bcf_gt_phased(majorAllele) : bcf_gt_unphased(majorAllele);
- }
-
- // replace gts
- if ( nbinom && ngts>=2 ) // only diploid genotypes are considered: higher ploidy ignored further, haploid here
- {
- if ( args->filter ) filter_test(args->filter,rec,(const uint8_t **)&args->smpl_pass);
- for (i=0; i<rec->n_sample; i++)
- {
- if ( args->smpl_pass )
- {
- if ( !args->smpl_pass[i] && args->filter_logic==FLT_INCLUDE ) continue;
- if ( args->smpl_pass[i] && args->filter_logic==FLT_EXCLUDE ) continue;
- }
- int32_t *ptr = args->gts + i*ngts;
- if ( bcf_gt_is_missing(ptr[0]) || bcf_gt_is_missing(ptr[1]) || ptr[1]==bcf_int32_vector_end ) continue;
- if ( ptr[0]==ptr[1] ) continue; // a hom
- int ia = bcf_gt_allele(ptr[0]);
- int ib = bcf_gt_allele(ptr[1]);
- if ( ia>=nbinom || ib>=nbinom )
- error("The sample %s has incorrect number of %s fields at %s:%d\n",
- args->in_hdr->samples[i],args->binom_tag,bcf_seqname(args->in_hdr,rec),rec->pos+1);
-
- double prob = calc_binom(args->iarr[i*nbinom+ia],args->iarr[i*nbinom+ib]);
- if ( !args->binom_cmp(prob,args->binom_val) ) continue;
-
- if ( args->new_mask>_UNPHASED )
- changed += unphase_gt(ptr, ngts);
- else if ( args->new_mask==GT_PHASED )
- changed += phase_gt(ptr, ngts);
- else
- changed += set_gt(ptr, ngts, args->new_gt);
- }
- }
- else if ( args->tgt_mask>_QUERY )
- {
- int pass_site = filter_test(args->filter,rec,(const uint8_t **)&args->smpl_pass);
- if ( pass_site && args->filter_logic==FLT_EXCLUDE )
- {
- // -i can include a site but exclude a sample, -e exclude a site but include a sample
- if ( pass_site )
- {
- if ( !args->smpl_pass ) return rec;
- pass_site = 0;
- for (i=0; i<rec->n_sample; i++)
- {
- if ( args->smpl_pass[i] ) args->smpl_pass[i] = 0;
- else { args->smpl_pass[i] = 1; pass_site = 1; }
- }
- if ( !pass_site ) return rec;
- }
- else if ( args->smpl_pass )
- for (i=0; i<rec->n_sample; i++) args->smpl_pass[i] = 1;
- }
- else if ( !pass_site ) return rec;
-
- for (i=0; i<rec->n_sample; i++)
- {
- if ( !args->smpl_pass[i] ) continue;
- if ( args->new_mask>_UNPHASED )
- changed += unphase_gt(args->gts + i*ngts, ngts);
- else if ( args->new_mask==GT_PHASED )
- changed += phase_gt(args->gts + i*ngts, ngts);
- else
- changed += set_gt(args->gts + i*ngts, ngts, args->new_gt);
- }
- }
- else
- {
- for (i=0; i<rec->n_sample; i++)
- {
- int ploidy = 0, nmiss = 0;
- int32_t *ptr = args->gts + i*ngts;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- ploidy++;
- if ( ptr[j]==bcf_gt_missing ) nmiss++;
- }
-
- int do_set = 0;
- if ( args->tgt_mask>_ALL ) do_set = 1;
- else if ( args->tgt_mask>_PARTIAL && nmiss ) do_set = 1;
- else if ( args->tgt_mask>_MISSING && ploidy==nmiss ) do_set = 1;
-
- if ( !do_set ) continue;
-
- if ( args->new_mask>_UNPHASED )
- changed += unphase_gt(ptr, ngts);
- else if ( args->new_mask==GT_PHASED )
- changed += phase_gt(ptr, ngts);
- else
- changed += set_gt(ptr, ngts, args->new_gt);
- }
- }
- args->nchanged += changed;
- if ( changed ) bcf_update_genotypes(args->out_hdr, rec, args->gts, ngts*rec->n_sample);
- return rec;
-}
-
-void destroy(void)
-{
- fprintf(stderr,"Filled %"PRId64" alleles\n", args->nchanged);
- free(args->binom_tag);
- if ( args->filter ) filter_destroy(args->filter);
- free(args->arr);
- free(args->iarr);
- free(args->gts);
- free(args);
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugins/setGT.c -- set gentoypes to given values
-
- Copyright (C) 2015-2017 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <htslib/vcf.h>
-#include <htslib/vcfutils.h>
-#include <htslib/kfunc.h>
-#include <inttypes.h>
-#include <getopt.h>
-#include <ctype.h>
-#include "bcftools.h"
-#include "filter.h"
-
-// Logic of the filters: include or exclude sites which match the filters?
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-typedef int (*cmp_f)(double a, double b);
-
-static int cmp_eq(double a, double b) { return a==b ? 1 : 0; }
-static int cmp_le(double a, double b) { return a<=b ? 1 : 0; }
-static int cmp_ge(double a, double b) { return a>=b ? 1 : 0; }
-static int cmp_lt(double a, double b) { return a<b ? 1 : 0; }
-static int cmp_gt(double a, double b) { return a>b ? 1 : 0; }
-
-typedef struct
-{
- bcf_hdr_t *in_hdr, *out_hdr;
- int32_t *gts, mgts, *iarr, miarr;
- int *arr, marr;
- uint64_t nchanged;
- int tgt_mask, new_mask, new_gt;
- filter_t *filter;
- char *filter_str;
- int filter_logic;
- uint8_t *smpl_pass;
- double binom_val;
- char *binom_tag;
- cmp_f binom_cmp;
-}
-args_t;
-
-args_t *args = NULL;
-
-#define GT_MISSING 1
-#define GT_PARTIAL (1<<1)
-#define GT_REF (1<<2)
-#define GT_MAJOR (1<<3)
-#define GT_PHASED (1<<4)
-#define GT_UNPHASED (1<<5)
-#define GT_ALL (1<<6)
-#define GT_QUERY (1<<7)
-#define GT_BINOM (1<<8)
-
-const char *about(void)
-{
- return "Set genotypes: partially missing to missing, missing to ref/major allele, etc.\n";
-}
-
-const char *usage(void)
-{
- return
- "About: Sets genotypes. The target genotypes can be specified as:\n"
- " ./. .. completely missing (\".\" or \"./.\", depending on ploidy)\n"
- " ./x .. partially missing (e.g., \"./0\" or \".|1\" but not \"./.\")\n"
- " . .. partially or completely missing\n"
- " a .. all genotypes\n"
- " b .. heterozygous genotypes failing two-tailed binomial test (example below)\n"
- " q .. select genotypes using -i/-e options\n"
- " and the new genotype can be one of:\n"
- " . .. missing (\".\" or \"./.\", keeps ploidy)\n"
- " 0 .. reference allele\n"
- " M .. major allele\n"
- " p .. phased genotype\n"
- " u .. unphase genotype and sort by allele (1|0 becomes 0/1)\n"
- "Usage: bcftools +setGT [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -e, --exclude <expr> Exclude a genotype if true (requires -t q)\n"
- " -i, --include <expr> include a genotype if true (requires -t q)\n"
- " -n, --new-gt <type> Genotypes to set, see above\n"
- " -t, --target-gt <type> Genotypes to change, see above\n"
- "\n"
- "Example:\n"
- " # set missing genotypes (\"./.\") to phased ref genotypes (\"0|0\")\n"
- " bcftools +setGT in.vcf -- -t . -n 0p\n"
- "\n"
- " # set missing genotypes with DP>0 and GQ>20 to ref genotypes (\"0/0\")\n"
- " bcftools +setGT in.vcf -- -t q -n 0 -i 'GT=\".\" && FMT/DP>0 && GQ>20'\n"
- "\n"
- " # set partially missing genotypes to completely missing\n"
- " bcftools +setGT in.vcf -- -t ./x -n .\n"
- "\n"
- " # set heterozygous genotypes to 0/0 if binom.test(nAlt,nRef+nAlt,0.5)<1e-3\n"
- " bcftools +setGT in.vcf -- -t \"b:AD<1e-3\" -n 0\n" // todo: make -i/-e recognise something like is_het or gt="het" so that this can be generalized?
- "\n";
-}
-
-static void _parse_binom_expr_error(char *str)
-{
- error(
- "Error parsing the expression: %s\n"
- "Expected TAG CMP VAL, where\n"
- " TAG .. one of the format tags\n"
- " CMP .. operator, one of <, <=, >, >=\n"
- " VAL .. value\n"
- "For example:\n"
- " bcftools +setGT in.vcf -- -t \"b:AD>1e-3\" -n 0\n"
- "\n", str
- );
-}
-void parse_binom_expr(args_t *args, char *str)
-{
- if ( str[1]!=':' ) _parse_binom_expr_error(str);
-
- char *beg = str+2;
- while ( *beg && isspace(*beg) ) beg++;
- if ( !*beg ) _parse_binom_expr_error(str);
- char *end = beg;
- while ( *end )
- {
- if ( isspace(*end) || *end=='<' || *end=='=' || *end=='>' ) break;
- end++;
- }
- if ( !*end ) _parse_binom_expr_error(str);
- args->binom_tag = (char*) calloc(1,end-beg+1);
- memcpy(args->binom_tag,beg,end-beg);
- int tag_id = bcf_hdr_id2int(args->in_hdr,BCF_DT_ID,args->binom_tag);
- if ( !bcf_hdr_idinfo_exists(args->in_hdr,BCF_HL_FMT,tag_id) ) error("The FORMAT tag \"%s\" is not present in the VCF\n", args->binom_tag);
-
- while ( *end && isspace(*end) ) end++;
- if ( !*end ) _parse_binom_expr_error(str);
-
- if ( !strncmp(end,"<=",2) ) { args->binom_cmp = cmp_le; beg = end+2; }
- else if ( !strncmp(end,">=",2) ) { args->binom_cmp = cmp_ge; beg = end+2; }
- else if ( !strncmp(end,"==",2) ) { args->binom_cmp = cmp_eq; beg = end+2; }
- else if ( !strncmp(end,"<",1) ) { args->binom_cmp = cmp_lt; beg = end+1; }
- else if ( !strncmp(end,">",1) ) { args->binom_cmp = cmp_gt; beg = end+1; }
- else if ( !strncmp(end,"=",1) ) { args->binom_cmp = cmp_eq; beg = end+1; }
- else _parse_binom_expr_error(str);
-
- while ( *beg && isspace(*beg) ) beg++;
- if ( !*beg ) _parse_binom_expr_error(str);
-
- args->binom_val = strtod(beg, &end);
- while ( *end && isspace(*end) ) end++;
- if ( *end ) _parse_binom_expr_error(str);
-
- args->tgt_mask |= GT_BINOM;
- return;
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- args = (args_t*) calloc(1,sizeof(args_t));
- args->in_hdr = in;
- args->out_hdr = out;
-
- int c;
- static struct option loptions[] =
- {
- {"include",required_argument,NULL,'i'},
- {"exclude",required_argument,NULL,'e'},
- {"new-gt",required_argument,NULL,'n'},
- {"target-gt",required_argument,NULL,'t'},
- {NULL,0,NULL,0}
- };
- while ((c = getopt_long(argc, argv, "?hn:t:i:e:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'i': args->filter_str = optarg; args->filter_logic = FLT_INCLUDE; break;
- case 'e': args->filter_str = optarg; args->filter_logic = FLT_EXCLUDE; break;
- case 'n': args->new_mask = 0;
- if ( strchr(optarg,'.') ) args->new_mask |= GT_MISSING;
- if ( strchr(optarg,'0') ) args->new_mask |= GT_REF;
- if ( strchr(optarg,'M') ) args->new_mask |= GT_MAJOR;
- if ( strchr(optarg,'p') ) args->new_mask |= GT_PHASED;
- if ( strchr(optarg,'u') ) args->new_mask |= GT_UNPHASED;
- if ( args->new_mask==0 ) error("Unknown parameter to --new-gt: %s\n", optarg);
- break;
- case 't':
- if ( !strcmp(optarg,".") ) args->tgt_mask |= GT_MISSING|GT_PARTIAL;
- if ( !strcmp(optarg,"./x") ) args->tgt_mask |= GT_PARTIAL;
- if ( !strcmp(optarg,"./.") ) args->tgt_mask |= GT_MISSING;
- if ( !strcmp(optarg,"a") ) args->tgt_mask |= GT_ALL;
- if ( !strcmp(optarg,"q") ) args->tgt_mask |= GT_QUERY;
- if ( !strcmp(optarg,"?") ) args->tgt_mask |= GT_QUERY; // for backward compatibility
- if ( strchr(optarg,'b') ) parse_binom_expr(args, strchr(optarg,'b'));
- if ( args->tgt_mask==0 ) error("Unknown parameter to --target-gt: %s\n", optarg);
- break;
- case 'h':
- case '?':
- default: fprintf(bcftools_stderr,"%s", usage()); exit(1); break;
- }
- }
-
- if ( !args->new_mask ) error("Expected -n option\n");
- if ( !args->tgt_mask ) error("Expected -t option\n");
-
- if ( args->new_mask & GT_MISSING ) args->new_gt = bcf_gt_missing;
- if ( args->new_mask & GT_REF ) args->new_gt = args->new_mask>_PHASED ? bcf_gt_phased(0) : bcf_gt_unphased(0);
-
- if ( args->filter_str && !(args->tgt_mask>_QUERY) ) error("Expected -tq with -i/-e\n");
- if ( !args->filter_str && args->tgt_mask>_QUERY ) error("Expected -i/-e with -tq\n");
- if ( args->filter_str ) args->filter = filter_init(in,args->filter_str);
-
- return 0;
-}
-
-static inline int phase_gt(int32_t *ptr, int ngts)
-{
- int j, changed = 0;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( bcf_gt_is_phased(ptr[j]) ) continue;
- ptr[j] = bcf_gt_phased(bcf_gt_allele(ptr[j])); // add phasing; this may need a fix, I think the flag should be set for one allele only?
- changed++;
- }
- return changed;
-}
-
-static inline int unphase_gt(int32_t *ptr, int ngts)
-{
- int j, changed = 0;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( !bcf_gt_is_phased(ptr[j]) ) continue;
- ptr[j] = bcf_gt_unphased(bcf_gt_allele(ptr[j])); // remove phasing
- changed++;
- }
-
- // insertion sort
- int k, l;
- for (k=1; k<j; k++)
- {
- int32_t x = ptr[k];
- l = k;
- while ( l>0 && ptr[l-1]>x )
- {
- ptr[l] = ptr[l-1];
- l--;
- }
- ptr[l] = x;
- }
- return changed;
-}
-static inline int set_gt(int32_t *ptr, int ngts, int gt)
-{
- int j, changed = 0;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- if ( ptr[j] != gt ) changed++;
- ptr[j] = gt;
- }
- return changed;
-}
-
-static inline double calc_binom(int na, int nb)
-{
- if ( na + nb == 0 ) return 1;
-
- /*
- kfunc.h implements kf_betai, which is the regularized beta function I_x(a,b) = P(X<=a/(a+b))
- */
- double prob = na > nb ? 2*kf_betai(na, nb + 1, 0.5) : 2*kf_betai(nb, na + 1, 0.5);
- if ( prob > 1 ) prob = 1;
-
- return prob;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- if ( !rec->n_sample ) return rec;
-
- int ngts = bcf_get_genotypes(args->in_hdr, rec, &args->gts, &args->mgts);
- ngts /= rec->n_sample;
- int i, j, changed = 0;
-
- int nbinom = 0;
- if ( args->tgt_mask & GT_BINOM )
- {
- nbinom = bcf_get_format_int32(args->in_hdr, rec, args->binom_tag, &args->iarr, &args->miarr);
- if ( nbinom<0 ) nbinom = 0;
- nbinom /= rec->n_sample;
- }
-
- // Calculating allele frequency for each allele and determining major allele
- // only do this if use_major is true
- int an = 0, maxAC = -1, majorAllele = -1;
- if ( args->new_mask & GT_MAJOR )
- {
- hts_expand(int,rec->n_allele,args->marr,args->arr);
- int ret = bcf_calc_ac(args->in_hdr,rec,args->arr,BCF_UN_FMT);
- if ( ret<= 0 )
- error("Could not calculate allele count at %s:%d\n", bcf_seqname(args->in_hdr,rec),rec->pos+1);
-
- for(i=0; i < rec->n_allele; ++i)
- {
- an += args->arr[i];
- if (args->arr[i] > maxAC)
- {
- maxAC = args->arr[i];
- majorAllele = i;
- }
- }
-
- // replacing new_gt by major allele
- args->new_gt = args->new_mask & GT_PHASED ? bcf_gt_phased(majorAllele) : bcf_gt_unphased(majorAllele);
- }
-
- // replace gts
- if ( nbinom && ngts>=2 ) // only diploid genotypes are considered: higher ploidy ignored further, haploid here
- {
- if ( args->filter ) filter_test(args->filter,rec,(const uint8_t **)&args->smpl_pass);
- for (i=0; i<rec->n_sample; i++)
- {
- if ( args->smpl_pass )
- {
- if ( !args->smpl_pass[i] && args->filter_logic==FLT_INCLUDE ) continue;
- if ( args->smpl_pass[i] && args->filter_logic==FLT_EXCLUDE ) continue;
- }
- int32_t *ptr = args->gts + i*ngts;
- if ( bcf_gt_is_missing(ptr[0]) || bcf_gt_is_missing(ptr[1]) || ptr[1]==bcf_int32_vector_end ) continue;
- if ( ptr[0]==ptr[1] ) continue; // a hom
- int ia = bcf_gt_allele(ptr[0]);
- int ib = bcf_gt_allele(ptr[1]);
- if ( ia>=nbinom || ib>=nbinom )
- error("The sample %s has incorrect number of %s fields at %s:%d\n",
- args->in_hdr->samples[i],args->binom_tag,bcf_seqname(args->in_hdr,rec),rec->pos+1);
-
- double prob = calc_binom(args->iarr[i*nbinom+ia],args->iarr[i*nbinom+ib]);
- if ( !args->binom_cmp(prob,args->binom_val) ) continue;
-
- if ( args->new_mask>_UNPHASED )
- changed += unphase_gt(ptr, ngts);
- else if ( args->new_mask==GT_PHASED )
- changed += phase_gt(ptr, ngts);
- else
- changed += set_gt(ptr, ngts, args->new_gt);
- }
- }
- else if ( args->tgt_mask>_QUERY )
- {
- int pass_site = filter_test(args->filter,rec,(const uint8_t **)&args->smpl_pass);
- if ( pass_site && args->filter_logic==FLT_EXCLUDE )
- {
- // -i can include a site but exclude a sample, -e exclude a site but include a sample
- if ( pass_site )
- {
- if ( !args->smpl_pass ) return rec;
- pass_site = 0;
- for (i=0; i<rec->n_sample; i++)
- {
- if ( args->smpl_pass[i] ) args->smpl_pass[i] = 0;
- else { args->smpl_pass[i] = 1; pass_site = 1; }
- }
- if ( !pass_site ) return rec;
- }
- else if ( args->smpl_pass )
- for (i=0; i<rec->n_sample; i++) args->smpl_pass[i] = 1;
- }
- else if ( !pass_site ) return rec;
-
- for (i=0; i<rec->n_sample; i++)
- {
- if ( !args->smpl_pass[i] ) continue;
- if ( args->new_mask>_UNPHASED )
- changed += unphase_gt(args->gts + i*ngts, ngts);
- else if ( args->new_mask==GT_PHASED )
- changed += phase_gt(args->gts + i*ngts, ngts);
- else
- changed += set_gt(args->gts + i*ngts, ngts, args->new_gt);
- }
- }
- else
- {
- for (i=0; i<rec->n_sample; i++)
- {
- int ploidy = 0, nmiss = 0;
- int32_t *ptr = args->gts + i*ngts;
- for (j=0; j<ngts; j++)
- {
- if ( ptr[j]==bcf_int32_vector_end ) break;
- ploidy++;
- if ( ptr[j]==bcf_gt_missing ) nmiss++;
- }
-
- int do_set = 0;
- if ( args->tgt_mask>_ALL ) do_set = 1;
- else if ( args->tgt_mask>_PARTIAL && nmiss ) do_set = 1;
- else if ( args->tgt_mask>_MISSING && ploidy==nmiss ) do_set = 1;
-
- if ( !do_set ) continue;
-
- if ( args->new_mask>_UNPHASED )
- changed += unphase_gt(ptr, ngts);
- else if ( args->new_mask==GT_PHASED )
- changed += phase_gt(ptr, ngts);
- else
- changed += set_gt(ptr, ngts, args->new_gt);
- }
- }
- args->nchanged += changed;
- if ( changed ) bcf_update_genotypes(args->out_hdr, rec, args->gts, ngts*rec->n_sample);
- return rec;
-}
-
-void destroy(void)
-{
- fprintf(bcftools_stderr,"Filled %"PRId64" alleles\n", args->nchanged);
- free(args->binom_tag);
- if ( args->filter ) filter_destroy(args->filter);
- free(args->arr);
- free(args->iarr);
- free(args->gts);
- free(args);
-}
-
-
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2018 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <unistd.h> // for isatty
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include "bcftools.h"
-#include "filter.h"
-
-
-// Logic of the filters: include or exclude sites which match the filters?
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-typedef struct
-{
- uint32_t
- npass, // number of genotypes passing the filter
- nnon_ref, // number of non-reference genotypes
- nhomRR,
- nhomAA,
- nhemi,
- nhet,
- nSNV,
- nIndel,
- nmissing,
- nsingleton, // het different from everyone else
- nts, ntv; // number of transitions and transversions
-}
-stats_t;
-
-typedef struct
-{
- stats_t *stats, site_stats;
- filter_t *filter;
- char *expr;
-}
-flt_stats_t;
-
-typedef struct
-{
- int argc, filter_logic, regions_is_file, targets_is_file;
- int nflt_str;
- char *filter_str, **flt_str;
- char **argv, *output_fname, *fname, *regions, *targets;
- bcf_srs_t *sr;
- bcf_hdr_t *hdr;
- flt_stats_t *filters;
- int nfilters, nsmpl;
- int32_t *gt_arr, *ac;
- int mgt_arr, mac;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Calculate basic per-sample stats scanning over a range of thresholds simultaneously.\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Calculates basic per-sample stats. Use curly brackets to scan a range of values simultaneously\n"
- "Usage: bcftools +smpl-stats [Plugin Options]\n"
- "Plugin options:\n"
- " -e, --exclude EXPR exclude sites and samples for which the expression is true\n"
- " -i, --include EXPR include sites and samples for which the expression is true\n"
- " -o, --output FILE output file name [stdout]\n"
- " -r, --regions REG restrict to comma-separated list of regions\n"
- " -R, --regions-file FILE restrict to regions listed in a file\n"
- " -t, --targets REG similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file FILE similar to -R but streams rather than index-jumps\n"
- "\n"
- "Example:\n"
- " bcftools +smpl-stats -i 'GQ>{10,20,30,40,50}' file.bcf\n"
- "\n";
-}
-
-static void parse_filters(args_t *args)
-{
- if ( !args->filter_str ) return;
- int mflt = 1;
- args->nflt_str = 1;
- args->flt_str = (char**) malloc(sizeof(char*));
- args->flt_str[0] = strdup(args->filter_str);
- while (1)
- {
- int i, expanded = 0;
- for (i=args->nflt_str-1; i>=0; i--)
- {
- char *exp_beg = strchr(args->flt_str[i], '{');
- if ( !exp_beg ) continue;
- char *exp_end = strchr(exp_beg+1, '}');
- if ( !exp_end ) error("Could not parse the expression: %s\n", args->filter_str);
- char *beg = exp_beg+1, *mid = beg;
- while ( mid<exp_end )
- {
- while ( mid<exp_end && *mid!=',' ) mid++;
- kstring_t tmp = {0,0,0};
- kputsn(args->flt_str[i], exp_beg - args->flt_str[i], &tmp);
- kputsn(beg, mid - beg, &tmp);
- kputs(exp_end+1, &tmp);
- args->nflt_str++;
- hts_expand(char*, args->nflt_str, mflt, args->flt_str);
- args->flt_str[args->nflt_str-1] = tmp.s;
- beg = ++mid;
- }
- expanded = 1;
- free(args->flt_str[i]);
- memmove(&args->flt_str[i], &args->flt_str[i+1], (args->nflt_str-i-1)*sizeof(*args->flt_str));
- args->nflt_str--;
- args->flt_str[args->nflt_str] = NULL;
- }
- if ( !expanded ) break;
- }
-
- fprintf(stderr,"Collecting data for %d filtering expressions\n", args->nflt_str);
-}
-
-static void init_data(args_t *args)
-{
- args->sr = bcf_sr_init();
- if ( args->regions )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, args->regions, args->regions_is_file)<0 ) error("Failed to read the regions: %s\n",args->regions);
- }
- if ( args->targets && bcf_sr_set_targets(args->sr, args->targets, args->targets_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->targets);
- if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr = bcf_sr_get_header(args->sr,0);
-
- parse_filters(args);
-
- int i;
- if ( !args->nflt_str )
- {
- args->filters = (flt_stats_t*) calloc(1, sizeof(flt_stats_t));
- args->nfilters = 1;
- args->filters[0].expr = strdup("all");
- }
- else
- {
- args->nfilters = args->nflt_str;
- args->filters = (flt_stats_t*) calloc(args->nfilters, sizeof(flt_stats_t));
- for (i=0; i<args->nfilters; i++)
- {
- args->filters[i].filter = filter_init(args->hdr, args->flt_str[i]);
- args->filters[i].expr = strdup(args->flt_str[i]);
-
- // replace tab's with spaces so that the output stays parsable
- char *tmp = args->filters[i].expr;
- while ( *tmp )
- {
- if ( *tmp=='\t' ) *tmp = ' ';
- tmp++;
- }
- }
- }
- args->nsmpl = bcf_hdr_nsamples(args->hdr);
- for (i=0; i<args->nfilters; i++)
- args->filters[i].stats = (stats_t*) calloc(args->nsmpl,sizeof(stats_t));
-}
-static void destroy_data(args_t *args)
-{
- int i;
- for (i=0; i<args->nfilters; i++)
- {
- if ( args->filters[i].filter ) filter_destroy(args->filters[i].filter);
- free(args->filters[i].stats);
- free(args->filters[i].expr);
- }
- free(args->filters);
- for (i=0; i<args->nflt_str; i++) free(args->flt_str[i]);
- free(args->flt_str);
- bcf_sr_destroy(args->sr);
- free(args->ac);
- free(args->gt_arr);
- free(args);
-}
-static void report_stats(args_t *args)
-{
- int i = 0,j;
- FILE *fh = !args->output_fname || !strcmp("-",args->output_fname) ? stdout : fopen(args->output_fname,"w");
- if ( !fh ) error("Could not open the file for writing: %s\n", args->output_fname);
- fprintf(fh,"# CMD line shows the command line used to generate this output\n");
- fprintf(fh,"# DEF lines define expressions for all tested thresholds\n");
- fprintf(fh,"# FLT* lines report numbers for every threshold and every sample:\n");
- fprintf(fh,"# %d) filter id\n", ++i);
- fprintf(fh,"# %d) sample\n", ++i);
- fprintf(fh,"# %d) number of genotypes which pass the filter\n", ++i);
- fprintf(fh,"# %d) number of non-reference genotypes\n", ++i);
- fprintf(fh,"# %d) number of homozygous ref genotypes (0/0 or 0)\n", ++i);
- fprintf(fh,"# %d) number of homozygous alt genotypes (1/1, 2/2, etc)\n", ++i);
- fprintf(fh,"# %d) number of heterozygous genotypes (0/1, 1/2, etc)\n", ++i);
- fprintf(fh,"# %d) number of hemizygous genotypes (0, 1, etc)\n", ++i);
- fprintf(fh,"# %d) number of SNVs\n", ++i);
- fprintf(fh,"# %d) number of indels\n", ++i);
- fprintf(fh,"# %d) number of singletons\n", ++i);
- fprintf(fh,"# %d) number of missing genotypes (./., ., ./0, etc)\n", ++i);
- fprintf(fh,"# %d) number of transitions (genotypes such as \"1/2\" are counted twice)\n", ++i);
- fprintf(fh,"# %d) number of transversions (genotypes such as \"1/2\" are counted twice)\n", ++i);
- fprintf(fh,"# %d) overall ts/tv\n", ++i);
- i = 0;
- fprintf(fh,"# SITE* lines report numbers for every threshold and site:\n");
- fprintf(fh,"# %d) filter id\n", ++i);
- fprintf(fh,"# %d) number of sites which pass the filter\n", ++i);
- fprintf(fh,"# %d) number of SNVs\n", ++i);
- fprintf(fh,"# %d) number of indels\n", ++i);
- fprintf(fh,"# %d) number of singletons\n", ++i);
- fprintf(fh,"# %d) number of transitions (counted at most once at multiallelic sites)\n", ++i);
- fprintf(fh,"# %d) number of transversions (counted at most once at multiallelic sites)\n", ++i);
- fprintf(fh,"# %d) overall ts/tv\n", ++i);
- fprintf(fh, "CMD\t%s", args->argv[0]);
- for (i=1; i<args->argc; i++) fprintf(fh, " %s",args->argv[i]);
- fprintf(fh, "\n");
- for (i=0; i<args->nfilters; i++)
- {
- flt_stats_t *flt = &args->filters[i];
- fprintf(fh,"DEF\tFLT%d\t%s\n", i, flt->expr);
- }
- for (i=0; i<args->nfilters; i++)
- {
- flt_stats_t *flt = &args->filters[i];
- for (j=0; j<args->nsmpl; j++)
- {
- fprintf(fh,"FLT%d", i);
- fprintf(fh,"\t%s",args->hdr->samples[j]);
- stats_t *stats = &flt->stats[j];
- fprintf(fh,"\t%d", stats->npass);
- fprintf(fh,"\t%d", stats->nnon_ref);
- fprintf(fh,"\t%d", stats->nhomRR);
- fprintf(fh,"\t%d", stats->nhomAA);
- fprintf(fh,"\t%d", stats->nhet);
- fprintf(fh,"\t%d", stats->nhemi);
- fprintf(fh,"\t%d", stats->nSNV);
- fprintf(fh,"\t%d", stats->nIndel);
- fprintf(fh,"\t%d", stats->nsingleton);
- fprintf(fh,"\t%d", stats->nmissing);
- fprintf(fh,"\t%d", stats->nts);
- fprintf(fh,"\t%d", stats->ntv);
- fprintf(fh,"\t%.2f", stats->ntv ? (float)stats->nts/stats->ntv : INFINITY);
- fprintf(fh,"\n");
- }
- fprintf(fh,"SITE%d", i);
- stats_t *stats = &flt->site_stats;
- fprintf(fh,"\t%d", stats->npass);
- fprintf(fh,"\t%d", stats->nSNV);
- fprintf(fh,"\t%d", stats->nIndel);
- fprintf(fh,"\t%d", stats->nsingleton);
- fprintf(fh,"\t%d", stats->nts);
- fprintf(fh,"\t%d", stats->ntv);
- fprintf(fh,"\t%.2f", stats->ntv ? (float)stats->nts/stats->ntv : INFINITY);
- fprintf(fh,"\n");
- }
- if ( fclose(fh)!=0 ) error("Close failed: %s\n", (!args->output_fname || !strcmp("-",args->output_fname)) ? "stdout" : args->output_fname);
-}
-
-static inline int parse_genotype(int32_t *arr, int ngt1, int idx, int als[2])
-{
- int32_t *ptr = arr + ngt1 * idx;
- if ( bcf_gt_is_missing(ptr[0]) ) return -1;
- als[0] = bcf_gt_allele(ptr[0]);
-
- if ( ngt1==1 || ptr[1]==bcf_int32_vector_end ) { ptr[1] = ptr[0]; return -2; }
-
- if ( bcf_gt_is_missing(ptr[1]) ) return -1;
- als[1] = bcf_gt_allele(ptr[1]);
-
- return 0;
-}
-
-static void process_record(args_t *args, bcf1_t *rec, flt_stats_t *flt)
-{
- int i,j;
- uint8_t *smpl_pass = NULL;
-
- // Find out which trios pass and if the site passes
- if ( flt->filter )
- {
- int pass_site = filter_test(flt->filter, rec, (const uint8_t**) &smpl_pass);
- if ( args->filter_logic & FLT_EXCLUDE )
- {
- if ( pass_site )
- {
- if ( !smpl_pass ) return;
- pass_site = 0;
- for (i=0; i<args->nsmpl; i++)
- {
- if ( smpl_pass[i] ) smpl_pass[i] = 0;
- else { smpl_pass[i] = 1; pass_site = 1; }
- }
- if ( !pass_site ) return;
- }
- else
- for (i=0; i<args->nsmpl; i++) smpl_pass[i] = 1;
- }
- else if ( !pass_site ) return;
- }
-
- // Find out the allele counts. Try to use INFO/AC, if not present, determine from the genotypes
- hts_expand(int, rec->n_allele, args->mac, args->ac);
- if ( !bcf_calc_ac(args->hdr, rec, args->ac, BCF_UN_INFO|BCF_UN_FMT) ) return;
-
- // Get the genotypes
- int ngt = bcf_get_genotypes(args->hdr, rec, &args->gt_arr, &args->mgt_arr);
- if ( ngt<0 ) return;
- int ngt1 = ngt / rec->n_sample;
-
-
- // For ts/tv: numeric code of the reference allele, -1 for insertions
- int ref = !rec->d.allele[0][1] ? bcf_acgt2int(*rec->d.allele[0]) : -1;
-
- int star_allele = -1;
- for (i=1; i<rec->n_allele; i++)
- if ( !rec->d.allele[i][1] && rec->d.allele[i][0]=='*' ) { star_allele = i; break; }
-
- // Run the stats
- int site_pass = 0;
- int site_SNV = 0;
- int site_Indel = 0;
- int site_has_ts = 0;
- int site_has_tv = 0;
- int site_singleton = 0;
- for (i=0; i<args->nsmpl; i++)
- {
- if ( smpl_pass && !smpl_pass[i] ) continue;
- stats_t *stats = &flt->stats[i];
-
- // Determine the alternate allele and the genotypes, skip if any of the alleles is missing.
- int als[2];
- int ret = parse_genotype(args->gt_arr, ngt1, i, als);
- if ( ret==-1 ) { stats->nmissing++; continue; } // missing allele
- if ( ret==-2 ) stats->nhemi++;
- else if ( als[0]!=als[1] ) stats->nhet++;
- else if ( als[0]==0 ) stats->nhomRR++;
- else stats->nhomAA++;
-
- stats->npass++;
- site_pass = 1;
-
- // Is there an alternate allele other than *?
- int has_nonref = 0;
- for (j=0; j<2; j++)
- {
- if ( als[j]==star_allele ) continue;
- if ( als[j]==0 ) continue;
- has_nonref = 1;
- }
- if ( !has_nonref ) continue; // only ref or * in this genotype
-
- stats->nnon_ref++;
-
- // Calculate ts/tv, count SNPs, indels. It does the right thing and handles also HetAA genotypes
- {
- int has_ts = 0, has_tv = 0, has_snv = 0, has_indel = 0;
- for (j=0; j<2; j++)
- {
- if ( als[j]==0 || als[j]==star_allele ) continue;
- if ( als[j] >= rec->n_allele )
- error("The GT index is out of range at %s:%d in %s\n", bcf_seqname(args->hdr,rec),rec->pos+1,args->hdr->samples[j]);
-
- if ( args->ac[als[j]]==1 ) { stats->nsingleton++; site_singleton = 1; }
-
- int var_type = bcf_get_variant_type(rec, als[j]);
- if ( var_type==VCF_SNP || var_type==VCF_MNP )
- {
- int k = 0;
- while ( rec->d.allele[0][k] && rec->d.allele[als[j]][k] )
- {
- if ( rec->d.allele[0][k]==rec->d.allele[als[j]][k] ) { k++; continue; }
-
- int alt = bcf_acgt2int(rec->d.allele[als[j]][k]);
- if ( abs(ref-alt)==2 ) has_ts = 1;
- else has_tv = 1;
- has_snv = 1;
-
- k++;
- }
- }
- else if ( var_type==VCF_INDEL ) has_indel = 1;
- }
- if ( has_ts ) { stats->nts++; site_has_ts = 1; }
- if ( has_tv ) { stats->ntv++; site_has_tv = 1; }
- if ( has_snv ) { stats->nSNV++; site_SNV = 1; }
- if ( has_indel ) { stats->nIndel++; site_Indel = 1; }
- }
- }
- flt->site_stats.npass += site_pass;
- flt->site_stats.nSNV += site_SNV;
- flt->site_stats.nIndel += site_Indel;
- flt->site_stats.nts += site_has_ts;
- flt->site_stats.ntv += site_has_tv;
- flt->site_stats.nsingleton += site_singleton;
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->output_fname = "-";
- static struct option loptions[] =
- {
- {"include",required_argument,0,'i'},
- {"exclude",required_argument,0,'e'},
- {"output",required_argument,NULL,'o'},
- {"regions",1,0,'r'},
- {"regions-file",1,0,'R'},
- {"targets",1,0,'t'},
- {"targets-file",1,0,'T'},
- {NULL,0,NULL,0}
- };
- int c, i;
- while ((c = getopt_long(argc, argv, "o:s:i:e:r:R:t:T:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'e': args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 't': args->targets = optarg; break;
- case 'T': args->targets = optarg; args->targets_is_file = 1; break;
- case 'r': args->regions = optarg; break;
- case 'R': args->regions = optarg; args->regions_is_file = 1; break;
- case 'o': args->output_fname = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else { error("%s",usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s",usage_text());
- else args->fname = argv[optind];
-
- init_data(args);
-
- while ( bcf_sr_next_line(args->sr) )
- {
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- for (i=0; i<args->nfilters; i++)
- process_record(args, rec, &args->filters[i]);
- }
-
- report_stats(args);
- destroy_data(args);
-
- return 0;
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2018 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <unistd.h> // for isatty
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include "bcftools.h"
-#include "filter.h"
-
-
-// Logic of the filters: include or exclude sites which match the filters?
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-typedef struct
-{
- uint32_t
- npass, // number of genotypes passing the filter
- nnon_ref, // number of non-reference genotypes
- nhomRR,
- nhomAA,
- nhemi,
- nhet,
- nSNV,
- nIndel,
- nmissing,
- nsingleton, // het different from everyone else
- nts, ntv; // number of transitions and transversions
-}
-stats_t;
-
-typedef struct
-{
- stats_t *stats, site_stats;
- filter_t *filter;
- char *expr;
-}
-flt_stats_t;
-
-typedef struct
-{
- int argc, filter_logic, regions_is_file, targets_is_file;
- int nflt_str;
- char *filter_str, **flt_str;
- char **argv, *output_fname, *fname, *regions, *targets;
- bcf_srs_t *sr;
- bcf_hdr_t *hdr;
- flt_stats_t *filters;
- int nfilters, nsmpl;
- int32_t *gt_arr, *ac;
- int mgt_arr, mac;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Calculate basic per-sample stats scanning over a range of thresholds simultaneously.\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Calculates basic per-sample stats. Use curly brackets to scan a range of values simultaneously\n"
- "Usage: bcftools +smpl-stats [Plugin Options]\n"
- "Plugin options:\n"
- " -e, --exclude EXPR exclude sites and samples for which the expression is true\n"
- " -i, --include EXPR include sites and samples for which the expression is true\n"
- " -o, --output FILE output file name [bcftools_stdout]\n"
- " -r, --regions REG restrict to comma-separated list of regions\n"
- " -R, --regions-file FILE restrict to regions listed in a file\n"
- " -t, --targets REG similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file FILE similar to -R but streams rather than index-jumps\n"
- "\n"
- "Example:\n"
- " bcftools +smpl-stats -i 'GQ>{10,20,30,40,50}' file.bcf\n"
- "\n";
-}
-
-static void parse_filters(args_t *args)
-{
- if ( !args->filter_str ) return;
- int mflt = 1;
- args->nflt_str = 1;
- args->flt_str = (char**) malloc(sizeof(char*));
- args->flt_str[0] = strdup(args->filter_str);
- while (1)
- {
- int i, expanded = 0;
- for (i=args->nflt_str-1; i>=0; i--)
- {
- char *exp_beg = strchr(args->flt_str[i], '{');
- if ( !exp_beg ) continue;
- char *exp_end = strchr(exp_beg+1, '}');
- if ( !exp_end ) error("Could not parse the expression: %s\n", args->filter_str);
- char *beg = exp_beg+1, *mid = beg;
- while ( mid<exp_end )
- {
- while ( mid<exp_end && *mid!=',' ) mid++;
- kstring_t tmp = {0,0,0};
- kputsn(args->flt_str[i], exp_beg - args->flt_str[i], &tmp);
- kputsn(beg, mid - beg, &tmp);
- kputs(exp_end+1, &tmp);
- args->nflt_str++;
- hts_expand(char*, args->nflt_str, mflt, args->flt_str);
- args->flt_str[args->nflt_str-1] = tmp.s;
- beg = ++mid;
- }
- expanded = 1;
- free(args->flt_str[i]);
- memmove(&args->flt_str[i], &args->flt_str[i+1], (args->nflt_str-i-1)*sizeof(*args->flt_str));
- args->nflt_str--;
- args->flt_str[args->nflt_str] = NULL;
- }
- if ( !expanded ) break;
- }
-
- fprintf(bcftools_stderr,"Collecting data for %d filtering expressions\n", args->nflt_str);
-}
-
-static void init_data(args_t *args)
-{
- args->sr = bcf_sr_init();
- if ( args->regions )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, args->regions, args->regions_is_file)<0 ) error("Failed to read the regions: %s\n",args->regions);
- }
- if ( args->targets && bcf_sr_set_targets(args->sr, args->targets, args->targets_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->targets);
- if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr = bcf_sr_get_header(args->sr,0);
-
- parse_filters(args);
-
- int i;
- if ( !args->nflt_str )
- {
- args->filters = (flt_stats_t*) calloc(1, sizeof(flt_stats_t));
- args->nfilters = 1;
- args->filters[0].expr = strdup("all");
- }
- else
- {
- args->nfilters = args->nflt_str;
- args->filters = (flt_stats_t*) calloc(args->nfilters, sizeof(flt_stats_t));
- for (i=0; i<args->nfilters; i++)
- {
- args->filters[i].filter = filter_init(args->hdr, args->flt_str[i]);
- args->filters[i].expr = strdup(args->flt_str[i]);
-
- // replace tab's with spaces so that the output stays parsable
- char *tmp = args->filters[i].expr;
- while ( *tmp )
- {
- if ( *tmp=='\t' ) *tmp = ' ';
- tmp++;
- }
- }
- }
- args->nsmpl = bcf_hdr_nsamples(args->hdr);
- for (i=0; i<args->nfilters; i++)
- args->filters[i].stats = (stats_t*) calloc(args->nsmpl,sizeof(stats_t));
-}
-static void destroy_data(args_t *args)
-{
- int i;
- for (i=0; i<args->nfilters; i++)
- {
- if ( args->filters[i].filter ) filter_destroy(args->filters[i].filter);
- free(args->filters[i].stats);
- free(args->filters[i].expr);
- }
- free(args->filters);
- for (i=0; i<args->nflt_str; i++) free(args->flt_str[i]);
- free(args->flt_str);
- bcf_sr_destroy(args->sr);
- free(args->ac);
- free(args->gt_arr);
- free(args);
-}
-static void report_stats(args_t *args)
-{
- int i = 0,j;
- FILE *fh = !args->output_fname || !strcmp("-",args->output_fname) ? bcftools_stdout : fopen(args->output_fname,"w");
- if ( !fh ) error("Could not open the file for writing: %s\n", args->output_fname);
- fprintf(fh,"# CMD line shows the command line used to generate this output\n");
- fprintf(fh,"# DEF lines define expressions for all tested thresholds\n");
- fprintf(fh,"# FLT* lines report numbers for every threshold and every sample:\n");
- fprintf(fh,"# %d) filter id\n", ++i);
- fprintf(fh,"# %d) sample\n", ++i);
- fprintf(fh,"# %d) number of genotypes which pass the filter\n", ++i);
- fprintf(fh,"# %d) number of non-reference genotypes\n", ++i);
- fprintf(fh,"# %d) number of homozygous ref genotypes (0/0 or 0)\n", ++i);
- fprintf(fh,"# %d) number of homozygous alt genotypes (1/1, 2/2, etc)\n", ++i);
- fprintf(fh,"# %d) number of heterozygous genotypes (0/1, 1/2, etc)\n", ++i);
- fprintf(fh,"# %d) number of hemizygous genotypes (0, 1, etc)\n", ++i);
- fprintf(fh,"# %d) number of SNVs\n", ++i);
- fprintf(fh,"# %d) number of indels\n", ++i);
- fprintf(fh,"# %d) number of singletons\n", ++i);
- fprintf(fh,"# %d) number of missing genotypes (./., ., ./0, etc)\n", ++i);
- fprintf(fh,"# %d) number of transitions (genotypes such as \"1/2\" are counted twice)\n", ++i);
- fprintf(fh,"# %d) number of transversions (genotypes such as \"1/2\" are counted twice)\n", ++i);
- fprintf(fh,"# %d) overall ts/tv\n", ++i);
- i = 0;
- fprintf(fh,"# SITE* lines report numbers for every threshold and site:\n");
- fprintf(fh,"# %d) filter id\n", ++i);
- fprintf(fh,"# %d) number of sites which pass the filter\n", ++i);
- fprintf(fh,"# %d) number of SNVs\n", ++i);
- fprintf(fh,"# %d) number of indels\n", ++i);
- fprintf(fh,"# %d) number of singletons\n", ++i);
- fprintf(fh,"# %d) number of transitions (counted at most once at multiallelic sites)\n", ++i);
- fprintf(fh,"# %d) number of transversions (counted at most once at multiallelic sites)\n", ++i);
- fprintf(fh,"# %d) overall ts/tv\n", ++i);
- fprintf(fh, "CMD\t%s", args->argv[0]);
- for (i=1; i<args->argc; i++) fprintf(fh, " %s",args->argv[i]);
- fprintf(fh, "\n");
- for (i=0; i<args->nfilters; i++)
- {
- flt_stats_t *flt = &args->filters[i];
- fprintf(fh,"DEF\tFLT%d\t%s\n", i, flt->expr);
- }
- for (i=0; i<args->nfilters; i++)
- {
- flt_stats_t *flt = &args->filters[i];
- for (j=0; j<args->nsmpl; j++)
- {
- fprintf(fh,"FLT%d", i);
- fprintf(fh,"\t%s",args->hdr->samples[j]);
- stats_t *stats = &flt->stats[j];
- fprintf(fh,"\t%d", stats->npass);
- fprintf(fh,"\t%d", stats->nnon_ref);
- fprintf(fh,"\t%d", stats->nhomRR);
- fprintf(fh,"\t%d", stats->nhomAA);
- fprintf(fh,"\t%d", stats->nhet);
- fprintf(fh,"\t%d", stats->nhemi);
- fprintf(fh,"\t%d", stats->nSNV);
- fprintf(fh,"\t%d", stats->nIndel);
- fprintf(fh,"\t%d", stats->nsingleton);
- fprintf(fh,"\t%d", stats->nmissing);
- fprintf(fh,"\t%d", stats->nts);
- fprintf(fh,"\t%d", stats->ntv);
- fprintf(fh,"\t%.2f", stats->ntv ? (float)stats->nts/stats->ntv : INFINITY);
- fprintf(fh,"\n");
- }
- fprintf(fh,"SITE%d", i);
- stats_t *stats = &flt->site_stats;
- fprintf(fh,"\t%d", stats->npass);
- fprintf(fh,"\t%d", stats->nSNV);
- fprintf(fh,"\t%d", stats->nIndel);
- fprintf(fh,"\t%d", stats->nsingleton);
- fprintf(fh,"\t%d", stats->nts);
- fprintf(fh,"\t%d", stats->ntv);
- fprintf(fh,"\t%.2f", stats->ntv ? (float)stats->nts/stats->ntv : INFINITY);
- fprintf(fh,"\n");
- }
- if ( fclose(fh)!=0 ) error("Close failed: %s\n", (!args->output_fname || !strcmp("-",args->output_fname)) ? "bcftools_stdout" : args->output_fname);
-}
-
-static inline int parse_genotype(int32_t *arr, int ngt1, int idx, int als[2])
-{
- int32_t *ptr = arr + ngt1 * idx;
- if ( bcf_gt_is_missing(ptr[0]) ) return -1;
- als[0] = bcf_gt_allele(ptr[0]);
-
- if ( ngt1==1 || ptr[1]==bcf_int32_vector_end ) { ptr[1] = ptr[0]; return -2; }
-
- if ( bcf_gt_is_missing(ptr[1]) ) return -1;
- als[1] = bcf_gt_allele(ptr[1]);
-
- return 0;
-}
-
-static void process_record(args_t *args, bcf1_t *rec, flt_stats_t *flt)
-{
- int i,j;
- uint8_t *smpl_pass = NULL;
-
- // Find out which trios pass and if the site passes
- if ( flt->filter )
- {
- int pass_site = filter_test(flt->filter, rec, (const uint8_t**) &smpl_pass);
- if ( args->filter_logic & FLT_EXCLUDE )
- {
- if ( pass_site )
- {
- if ( !smpl_pass ) return;
- pass_site = 0;
- for (i=0; i<args->nsmpl; i++)
- {
- if ( smpl_pass[i] ) smpl_pass[i] = 0;
- else { smpl_pass[i] = 1; pass_site = 1; }
- }
- if ( !pass_site ) return;
- }
- else
- for (i=0; i<args->nsmpl; i++) smpl_pass[i] = 1;
- }
- else if ( !pass_site ) return;
- }
-
- // Find out the allele counts. Try to use INFO/AC, if not present, determine from the genotypes
- hts_expand(int, rec->n_allele, args->mac, args->ac);
- if ( !bcf_calc_ac(args->hdr, rec, args->ac, BCF_UN_INFO|BCF_UN_FMT) ) return;
-
- // Get the genotypes
- int ngt = bcf_get_genotypes(args->hdr, rec, &args->gt_arr, &args->mgt_arr);
- if ( ngt<0 ) return;
- int ngt1 = ngt / rec->n_sample;
-
-
- // For ts/tv: numeric code of the reference allele, -1 for insertions
- int ref = !rec->d.allele[0][1] ? bcf_acgt2int(*rec->d.allele[0]) : -1;
-
- int star_allele = -1;
- for (i=1; i<rec->n_allele; i++)
- if ( !rec->d.allele[i][1] && rec->d.allele[i][0]=='*' ) { star_allele = i; break; }
-
- // Run the stats
- int site_pass = 0;
- int site_SNV = 0;
- int site_Indel = 0;
- int site_has_ts = 0;
- int site_has_tv = 0;
- int site_singleton = 0;
- for (i=0; i<args->nsmpl; i++)
- {
- if ( smpl_pass && !smpl_pass[i] ) continue;
- stats_t *stats = &flt->stats[i];
-
- // Determine the alternate allele and the genotypes, skip if any of the alleles is missing.
- int als[2];
- int ret = parse_genotype(args->gt_arr, ngt1, i, als);
- if ( ret==-1 ) { stats->nmissing++; continue; } // missing allele
- if ( ret==-2 ) stats->nhemi++;
- else if ( als[0]!=als[1] ) stats->nhet++;
- else if ( als[0]==0 ) stats->nhomRR++;
- else stats->nhomAA++;
-
- stats->npass++;
- site_pass = 1;
-
- // Is there an alternate allele other than *?
- int has_nonref = 0;
- for (j=0; j<2; j++)
- {
- if ( als[j]==star_allele ) continue;
- if ( als[j]==0 ) continue;
- has_nonref = 1;
- }
- if ( !has_nonref ) continue; // only ref or * in this genotype
-
- stats->nnon_ref++;
-
- // Calculate ts/tv, count SNPs, indels. It does the right thing and handles also HetAA genotypes
- {
- int has_ts = 0, has_tv = 0, has_snv = 0, has_indel = 0;
- for (j=0; j<2; j++)
- {
- if ( als[j]==0 || als[j]==star_allele ) continue;
- if ( als[j] >= rec->n_allele )
- error("The GT index is out of range at %s:%d in %s\n", bcf_seqname(args->hdr,rec),rec->pos+1,args->hdr->samples[j]);
-
- if ( args->ac[als[j]]==1 ) { stats->nsingleton++; site_singleton = 1; }
-
- int var_type = bcf_get_variant_type(rec, als[j]);
- if ( var_type==VCF_SNP || var_type==VCF_MNP )
- {
- int k = 0;
- while ( rec->d.allele[0][k] && rec->d.allele[als[j]][k] )
- {
- if ( rec->d.allele[0][k]==rec->d.allele[als[j]][k] ) { k++; continue; }
-
- int alt = bcf_acgt2int(rec->d.allele[als[j]][k]);
- if ( abs(ref-alt)==2 ) has_ts = 1;
- else has_tv = 1;
- has_snv = 1;
-
- k++;
- }
- }
- else if ( var_type==VCF_INDEL ) has_indel = 1;
- }
- if ( has_ts ) { stats->nts++; site_has_ts = 1; }
- if ( has_tv ) { stats->ntv++; site_has_tv = 1; }
- if ( has_snv ) { stats->nSNV++; site_SNV = 1; }
- if ( has_indel ) { stats->nIndel++; site_Indel = 1; }
- }
- }
- flt->site_stats.npass += site_pass;
- flt->site_stats.nSNV += site_SNV;
- flt->site_stats.nIndel += site_Indel;
- flt->site_stats.nts += site_has_ts;
- flt->site_stats.ntv += site_has_tv;
- flt->site_stats.nsingleton += site_singleton;
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->output_fname = "-";
- static struct option loptions[] =
- {
- {"include",required_argument,0,'i'},
- {"exclude",required_argument,0,'e'},
- {"output",required_argument,NULL,'o'},
- {"regions",1,0,'r'},
- {"regions-file",1,0,'R'},
- {"targets",1,0,'t'},
- {"targets-file",1,0,'T'},
- {NULL,0,NULL,0}
- };
- int c, i;
- while ((c = getopt_long(argc, argv, "o:s:i:e:r:R:t:T:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'e': args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 't': args->targets = optarg; break;
- case 'T': args->targets = optarg; args->targets_is_file = 1; break;
- case 'r': args->regions = optarg; break;
- case 'R': args->regions = optarg; args->regions_is_file = 1; break;
- case 'o': args->output_fname = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else { error("%s",usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s",usage_text());
- else args->fname = argv[optind];
-
- init_data(args);
-
- while ( bcf_sr_next_line(args->sr) )
- {
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- for (i=0; i<args->nfilters; i++)
- process_record(args, rec, &args->filters[i]);
- }
-
- report_stats(args);
- destroy_data(args);
-
- return 0;
-}
+++ /dev/null
-/*
- Copyright (C) 2017 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-/*
- Split VCF by sample(s)
-
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <unistd.h>
-#include <stdint.h>
-#include <errno.h>
-#include <ctype.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include "bcftools.h"
-#include "filter.h"
-
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-typedef struct
-{
- htsFile **fh;
- filter_t *filter;
- char *filter_str;
- int filter_logic; // one of FLT_INCLUDE/FLT_EXCLUDE (-i or -e)
- uint8_t *info_tags, *fmt_tags;
- int ninfo_tags, minfo_tags, nfmt_tags, mfmt_tags, keep_info, keep_fmt;
- int argc, region_is_file, target_is_file, output_type;
- char **argv, *region, *target, *fname, *output_dir, *keep_tags, **bnames, *samples_fname;
- bcf_hdr_t *hdr_in, *hdr_out;
- bcf_srs_t *sr;
-}
-args_t;
-
-const char *about(void)
-{
- return "Split VCF by sample creating single-sample VCFs\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Split VCF by sample, creating single-sample VCFs.\n"
- "\n"
- "Usage: bcftools +split [Options]\n"
- "Plugin options:\n"
- " -e, --exclude EXPR exclude sites for which the expression is true (applied on the outputs)\n"
- " -i, --include EXPR include only sites for which the expression is true (applied on the outputs)\n"
- " -k, --keep-tags LIST list of tags to keep. By default all tags are preserved\n"
- " -o, --output DIR write output to the directory DIR\n"
- " -O, --output-type b|u|z|v b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n"
- " -r, --regions REGION restrict to comma-separated list of regions\n"
- " -R, --regions-file FILE restrict to regions listed in a file\n"
- " -S, --samples-file FILE list of samples to keep with second (optional) column for basename of the new file\n"
- " -t, --targets REGION similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file FILE similar to -R but streams rather than index-jumps\n"
- "Examples:\n"
- " # Split a VCF file\n"
- " bcftools +split input.bcf -Ob -o dir\n"
- "\n"
- " # Exclude sites with missing or hom-ref genotypes\n"
- " bcftools +split input.bcf -Ob -o dir -i'GT=\"alt\"'\n"
- "\n"
- " # Keep all INFO tags but only GT and PL in FORMAT\n"
- " bcftools +split input.bcf -Ob -o dir -k INFO,FMT/GT,PL\n"
- "\n"
- " # Keep all FORMAT tags but drop all INFO tags\n"
- " bcftools +split input.bcf -Ob -o dir -k FMT\n"
- "\n";
-}
-
-void mkdir_p(const char *fmt, ...);
-
-char **set_file_base_names(args_t *args)
-{
- int i, nsmpl = bcf_hdr_nsamples(args->hdr_in);
- char **fnames = (char**) calloc(nsmpl,sizeof(char*));
- if ( args->samples_fname )
- {
- kstring_t str = {0,0,0};
- int nsamples = 0;
- char **samples = NULL;
- samples = hts_readlines(args->samples_fname, &nsamples);
- for (i=0; i<nsamples; i++)
- {
- str.l = 0;
- int escaped = 0;
- char *ptr = samples[i];
- while ( *ptr )
- {
- if ( *ptr=='\\' && !escaped ) { escaped = 1; ptr++; continue; }
- if ( isspace(*ptr) && !escaped ) break;
- kputc(*ptr, &str);
- escaped = 0;
- ptr++;
- }
- int idx = bcf_hdr_id2int(args->hdr_in, BCF_DT_SAMPLE, str.s);
- if ( idx<0 )
- {
- fprintf(stderr,"Warning: The sample \"%s\" is not present in %s\n", str.s,args->fname);
- continue;
- }
- while ( *ptr && isspace(*ptr) ) ptr++;
- if ( !*ptr )
- {
- fnames[idx] = strdup(str.s);
- continue;
- }
- str.l = 0;
- while ( *ptr )
- {
- if ( *ptr=='\\' && !escaped ) { escaped = 1; ptr++; continue; }
- if ( isspace(*ptr) && !escaped ) break;
- kputc(*ptr, &str);
- escaped = 0;
- ptr++;
- }
- fnames[idx] = strdup(str.s);
- }
- for (i=0; i<nsamples; i++) free(samples[i]);
- free(samples);
- free(str.s);
- }
- else
- {
- for (i=0; i<nsmpl; i++)
- fnames[i] = strdup(args->hdr_in->samples[i]);
- }
- return fnames;
-}
-
-static void init_data(args_t *args)
-{
- args->sr = bcf_sr_init();
- if ( args->region )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, args->region, args->region_is_file)<0 ) error("Failed to read the regions: %s\n",args->region);
- }
- if ( args->target && bcf_sr_set_targets(args->sr, args->target, args->target_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->target);
- if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr_in = bcf_sr_get_header(args->sr,0);
- args->hdr_out = bcf_hdr_dup(args->hdr_in);
-
- if ( args->filter_str )
- args->filter = filter_init(args->hdr_in, args->filter_str);
-
- mkdir_p("%s/",args->output_dir);
-
- int i, nsmpl = bcf_hdr_nsamples(args->hdr_in);
- if ( !nsmpl ) error("No samples to split: %s\n", args->fname);
- args->fh = (htsFile**)calloc(nsmpl,sizeof(*args->fh));
- args->bnames = set_file_base_names(args);
- kstring_t str = {0,0,0};
- for (i=0; i<nsmpl; i++)
- {
- if ( !args->bnames[i] ) continue;
- str.l = 0;
- kputs(args->output_dir, &str);
- if ( str.s[str.l-1] != '/' ) kputc('/', &str);
- int k, l = str.l;
- kputs(args->bnames[i], &str);
- for (k=l; k<str.l; k++) if ( isspace(str.s[k]) ) str.s[k] = '_';
- if ( args->output_type & FT_BCF ) kputs(".bcf", &str);
- else if ( args->output_type & FT_GZ ) kputs(".vcf.gz", &str);
- else kputs(".vcf", &str);
- args->fh[i] = hts_open(str.s, hts_bcf_wmode(args->output_type));
- if ( args->fh[i] == NULL ) error("Can't write to \"%s\": %s\n", str.s, strerror(errno));
- bcf_hdr_nsamples(args->hdr_out) = 1;
- args->hdr_out->samples[0] = args->bnames[i];
- bcf_hdr_write(args->fh[i], args->hdr_out);
- }
- free(str.s);
-
- // parse tags
- int is_info = 0, is_fmt = 0;
- char *beg = args->keep_tags;
- while ( beg && *beg )
- {
- if ( !strncasecmp("INFO/",beg,5) ) { is_info = 1; is_fmt = 0; beg += 5; }
- else if ( !strcasecmp("INFO",beg) ) { args->keep_info = 1; break; }
- else if ( !strncasecmp("INFO,",beg,5) ) { args->keep_info = 1; beg += 5; continue; }
- else if ( !strncasecmp("FMT/",beg,4) ) { is_info = 0; is_fmt = 1; beg += 4; }
- else if ( !strncasecmp("FORMAT/",beg,7) ) { is_info = 0; is_fmt = 1; beg += 7; }
- else if ( !strcasecmp("FMT",beg) ) { args->keep_fmt = 1; break; }
- else if ( !strcasecmp("FORMAT",beg) ) { args->keep_fmt = 1; break; }
- else if ( !strncasecmp("FMT,",beg,4) ) { args->keep_fmt = 1; beg += 4; continue; }
- else if ( !strncasecmp("FORMAT,",beg,7) ) { args->keep_fmt = 1; beg += 7; continue; }
- char *end = beg;
- while ( *end && *end!=',' ) end++;
- char tmp = *end; *end = 0;
- int id = bcf_hdr_id2int(args->hdr_in, BCF_DT_ID, beg);
- beg = tmp ? end + 1 : end;
- if ( is_info && bcf_hdr_idinfo_exists(args->hdr_in,BCF_HL_INFO,id) )
- {
- if ( id >= args->ninfo_tags ) args->ninfo_tags = id + 1;
- hts_expand0(uint8_t, args->ninfo_tags, args->minfo_tags, args->info_tags);
- args->info_tags[id] = 1;
- }
- if ( is_fmt && bcf_hdr_idinfo_exists(args->hdr_in,BCF_HL_FMT,id) )
- {
- if ( id >= args->nfmt_tags ) args->nfmt_tags = id + 1;
- hts_expand0(uint8_t, args->nfmt_tags, args->mfmt_tags, args->fmt_tags);
- args->fmt_tags[id] = 1;
- }
- }
- if ( !args->keep_info && !args->keep_fmt && !args->ninfo_tags && !args->nfmt_tags )
- {
- args->keep_info = args->keep_fmt = 1;
- }
-}
-static void destroy_data(args_t *args)
-{
- free(args->info_tags);
- free(args->fmt_tags);
- if ( args->filter )
- filter_destroy(args->filter);
- int i, nsmpl = bcf_hdr_nsamples(args->hdr_in);
- for (i=0; i<nsmpl; i++)
- {
- if ( args->fh[i] && hts_close(args->fh[i])!=0 ) error("Error: close failed!\n");
- free(args->bnames[i]);
- }
- free(args->bnames);
- free(args->fh);
- bcf_sr_destroy(args->sr);
- bcf_hdr_destroy(args->hdr_out);
- free(args);
-}
-
-static bcf1_t *rec_set_info(args_t *args, bcf1_t *rec)
-{
- bcf1_t *out = bcf_init1();
- out->rid = rec->rid;
- out->pos = rec->pos;
- out->rlen = rec->rlen;
- out->qual = rec->qual;
- out->n_allele = rec->n_allele;
- out->n_sample = 1;
- if ( args->keep_info )
- {
- out->n_info = rec->n_info;
- out->shared.m = out->shared.l = rec->shared.l;
- out->shared.s = (char*) malloc(out->shared.l);
- memcpy(out->shared.s,rec->shared.s,out->shared.l);
- return out;
- }
-
- // build the BCF record
- kstring_t tmp = {0,0,0};
- char *ptr = rec->shared.s;
- kputsn_(ptr, rec->unpack_size[0], &tmp); ptr += rec->unpack_size[0]; // ID
- kputsn_(ptr, rec->unpack_size[1], &tmp); ptr += rec->unpack_size[1]; // REF+ALT
- kputsn_(ptr, rec->unpack_size[2], &tmp); // FILTER
- if ( args->ninfo_tags )
- {
- int i;
- for (i=0; i<rec->n_info; i++)
- {
- bcf_info_t *info = &rec->d.info[i];
- int id = info->key;
- if ( !args->info_tags[id] ) continue;
- kputsn_(info->vptr - info->vptr_off, info->vptr_len + info->vptr_off, &tmp);
- out->n_info++;
- }
- }
- out->shared.m = tmp.m;
- out->shared.s = tmp.s;
- out->shared.l = tmp.l;
- out->unpacked = 0;
- return out;
-}
-
-static bcf1_t *rec_set_format(args_t *args, bcf1_t *src, int ismpl, bcf1_t *dst)
-{
- dst->n_fmt = 0;
- kstring_t tmp = dst->indiv; tmp.l = 0;
- int i;
- for (i=0; i<src->n_fmt; i++)
- {
- bcf_fmt_t *fmt = &src->d.fmt[i];
- int id = fmt->id;
- if ( !args->keep_fmt && !args->fmt_tags[id] ) continue;
-
- bcf_enc_int1(&tmp, id);
- bcf_enc_size(&tmp, fmt->n, fmt->type);
- kputsn_(fmt->p + ismpl*fmt->size, fmt->size, &tmp);
-
- dst->n_fmt++;
- }
- dst->indiv = tmp;
- return dst;
-}
-
-static void process(args_t *args)
-{
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- bcf_unpack(rec, BCF_UN_ALL);
-
- int i, site_pass = 1;
- const uint8_t *smpl_pass = NULL;
- if ( args->filter )
- {
- site_pass = filter_test(args->filter, rec, &smpl_pass);
- if ( args->filter_logic & FLT_EXCLUDE ) site_pass = site_pass ? 0 : 1;
- }
- bcf1_t *out = NULL;
- for (i=0; i<rec->n_sample; i++)
- {
- if ( !args->fh[i] ) continue;
- if ( !smpl_pass && !site_pass ) continue;
- if ( smpl_pass )
- {
- int pass = args->filter_logic & FLT_EXCLUDE ? ( smpl_pass[i] ? 0 : 1) : smpl_pass[i];
- if ( !pass ) continue;
- }
- if ( !out ) out = rec_set_info(args, rec);
- rec_set_format(args, rec, i, out);
- bcf_write(args->fh[i], args->hdr_out, out);
- }
- if ( out ) bcf_destroy(out);
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->output_type = FT_VCF;
- static struct option loptions[] =
- {
- {"keep-tags",required_argument,NULL,'k'},
- {"exclude",required_argument,NULL,'e'},
- {"include",required_argument,NULL,'i'},
- {"regions",required_argument,NULL,'r'},
- {"regions-file",required_argument,NULL,'R'},
- {"samples-file",required_argument,NULL,'S'},
- {"output",required_argument,NULL,'o'},
- {"output-type",required_argument,NULL,'O'},
- {NULL,0,NULL,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "vr:R:t:T:o:O:i:e:k:S:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'k': args->keep_tags = optarg; break;
- case 'e': args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 'T': args->target = optarg; args->target_is_file = 1; break;
- case 't': args->target = optarg; break;
- case 'R': args->region = optarg; args->region_is_file = 1; break;
- case 'S': args->samples_fname = optarg; break;
- case 'r': args->region = optarg; break;
- case 'o': args->output_dir = optarg; break;
- case 'O':
- switch (optarg[0]) {
- case 'b': args->output_type = FT_BCF_GZ; break;
- case 'u': args->output_type = FT_BCF; break;
- case 'z': args->output_type = FT_VCF_GZ; break;
- case 'v': args->output_type = FT_VCF; break;
- default: error("The output type \"%s\" not recognised\n", optarg);
- }
- break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else { error("%s", usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s", usage_text());
- else args->fname = argv[optind];
-
- if ( !args->output_dir ) error("Missing the -o option\n");
- if ( args->filter_logic == (FLT_EXCLUDE|FLT_INCLUDE) ) error("Only one of -i or -e can be given.\n");
-
- init_data(args);
-
- while ( bcf_sr_next_line(args->sr) ) process(args);
-
- destroy_data(args);
- return 0;
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/*
- Copyright (C) 2017 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-/*
- Split VCF by sample(s)
-
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <getopt.h>
-#include <stdarg.h>
-#include <unistd.h>
-#include <stdint.h>
-#include <errno.h>
-#include <ctype.h>
-#include <htslib/vcf.h>
-#include <htslib/synced_bcf_reader.h>
-#include "bcftools.h"
-#include "filter.h"
-
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-typedef struct
-{
- htsFile **fh;
- filter_t *filter;
- char *filter_str;
- int filter_logic; // one of FLT_INCLUDE/FLT_EXCLUDE (-i or -e)
- uint8_t *info_tags, *fmt_tags;
- int ninfo_tags, minfo_tags, nfmt_tags, mfmt_tags, keep_info, keep_fmt;
- int argc, region_is_file, target_is_file, output_type;
- char **argv, *region, *target, *fname, *output_dir, *keep_tags, **bnames, *samples_fname;
- bcf_hdr_t *hdr_in, *hdr_out;
- bcf_srs_t *sr;
-}
-args_t;
-
-const char *about(void)
-{
- return "Split VCF by sample creating single-sample VCFs\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Split VCF by sample, creating single-sample VCFs.\n"
- "\n"
- "Usage: bcftools +split [Options]\n"
- "Plugin options:\n"
- " -e, --exclude EXPR exclude sites for which the expression is true (applied on the outputs)\n"
- " -i, --include EXPR include only sites for which the expression is true (applied on the outputs)\n"
- " -k, --keep-tags LIST list of tags to keep. By default all tags are preserved\n"
- " -o, --output DIR write output to the directory DIR\n"
- " -O, --output-type b|u|z|v b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n"
- " -r, --regions REGION restrict to comma-separated list of regions\n"
- " -R, --regions-file FILE restrict to regions listed in a file\n"
- " -S, --samples-file FILE list of samples to keep with second (optional) column for basename of the new file\n"
- " -t, --targets REGION similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file FILE similar to -R but streams rather than index-jumps\n"
- "Examples:\n"
- " # Split a VCF file\n"
- " bcftools +split input.bcf -Ob -o dir\n"
- "\n"
- " # Exclude sites with missing or hom-ref genotypes\n"
- " bcftools +split input.bcf -Ob -o dir -i'GT=\"alt\"'\n"
- "\n"
- " # Keep all INFO tags but only GT and PL in FORMAT\n"
- " bcftools +split input.bcf -Ob -o dir -k INFO,FMT/GT,PL\n"
- "\n"
- " # Keep all FORMAT tags but drop all INFO tags\n"
- " bcftools +split input.bcf -Ob -o dir -k FMT\n"
- "\n";
-}
-
-void mkdir_p(const char *fmt, ...);
-
-char **set_file_base_names(args_t *args)
-{
- int i, nsmpl = bcf_hdr_nsamples(args->hdr_in);
- char **fnames = (char**) calloc(nsmpl,sizeof(char*));
- if ( args->samples_fname )
- {
- kstring_t str = {0,0,0};
- int nsamples = 0;
- char **samples = NULL;
- samples = hts_readlines(args->samples_fname, &nsamples);
- for (i=0; i<nsamples; i++)
- {
- str.l = 0;
- int escaped = 0;
- char *ptr = samples[i];
- while ( *ptr )
- {
- if ( *ptr=='\\' && !escaped ) { escaped = 1; ptr++; continue; }
- if ( isspace(*ptr) && !escaped ) break;
- kputc(*ptr, &str);
- escaped = 0;
- ptr++;
- }
- int idx = bcf_hdr_id2int(args->hdr_in, BCF_DT_SAMPLE, str.s);
- if ( idx<0 )
- {
- fprintf(bcftools_stderr,"Warning: The sample \"%s\" is not present in %s\n", str.s,args->fname);
- continue;
- }
- while ( *ptr && isspace(*ptr) ) ptr++;
- if ( !*ptr )
- {
- fnames[idx] = strdup(str.s);
- continue;
- }
- str.l = 0;
- while ( *ptr )
- {
- if ( *ptr=='\\' && !escaped ) { escaped = 1; ptr++; continue; }
- if ( isspace(*ptr) && !escaped ) break;
- kputc(*ptr, &str);
- escaped = 0;
- ptr++;
- }
- fnames[idx] = strdup(str.s);
- }
- for (i=0; i<nsamples; i++) free(samples[i]);
- free(samples);
- free(str.s);
- }
- else
- {
- for (i=0; i<nsmpl; i++)
- fnames[i] = strdup(args->hdr_in->samples[i]);
- }
- return fnames;
-}
-
-static void init_data(args_t *args)
-{
- args->sr = bcf_sr_init();
- if ( args->region )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, args->region, args->region_is_file)<0 ) error("Failed to read the regions: %s\n",args->region);
- }
- if ( args->target && bcf_sr_set_targets(args->sr, args->target, args->target_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->target);
- if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr_in = bcf_sr_get_header(args->sr,0);
- args->hdr_out = bcf_hdr_dup(args->hdr_in);
-
- if ( args->filter_str )
- args->filter = filter_init(args->hdr_in, args->filter_str);
-
- mkdir_p("%s/",args->output_dir);
-
- int i, nsmpl = bcf_hdr_nsamples(args->hdr_in);
- if ( !nsmpl ) error("No samples to split: %s\n", args->fname);
- args->fh = (htsFile**)calloc(nsmpl,sizeof(*args->fh));
- args->bnames = set_file_base_names(args);
- kstring_t str = {0,0,0};
- for (i=0; i<nsmpl; i++)
- {
- if ( !args->bnames[i] ) continue;
- str.l = 0;
- kputs(args->output_dir, &str);
- if ( str.s[str.l-1] != '/' ) kputc('/', &str);
- int k, l = str.l;
- kputs(args->bnames[i], &str);
- for (k=l; k<str.l; k++) if ( isspace(str.s[k]) ) str.s[k] = '_';
- if ( args->output_type & FT_BCF ) kputs(".bcf", &str);
- else if ( args->output_type & FT_GZ ) kputs(".vcf.gz", &str);
- else kputs(".vcf", &str);
- args->fh[i] = hts_open(str.s, hts_bcf_wmode(args->output_type));
- if ( args->fh[i] == NULL ) error("Can't write to \"%s\": %s\n", str.s, strerror(errno));
- bcf_hdr_nsamples(args->hdr_out) = 1;
- args->hdr_out->samples[0] = args->bnames[i];
- bcf_hdr_write(args->fh[i], args->hdr_out);
- }
- free(str.s);
-
- // parse tags
- int is_info = 0, is_fmt = 0;
- char *beg = args->keep_tags;
- while ( beg && *beg )
- {
- if ( !strncasecmp("INFO/",beg,5) ) { is_info = 1; is_fmt = 0; beg += 5; }
- else if ( !strcasecmp("INFO",beg) ) { args->keep_info = 1; break; }
- else if ( !strncasecmp("INFO,",beg,5) ) { args->keep_info = 1; beg += 5; continue; }
- else if ( !strncasecmp("FMT/",beg,4) ) { is_info = 0; is_fmt = 1; beg += 4; }
- else if ( !strncasecmp("FORMAT/",beg,7) ) { is_info = 0; is_fmt = 1; beg += 7; }
- else if ( !strcasecmp("FMT",beg) ) { args->keep_fmt = 1; break; }
- else if ( !strcasecmp("FORMAT",beg) ) { args->keep_fmt = 1; break; }
- else if ( !strncasecmp("FMT,",beg,4) ) { args->keep_fmt = 1; beg += 4; continue; }
- else if ( !strncasecmp("FORMAT,",beg,7) ) { args->keep_fmt = 1; beg += 7; continue; }
- char *end = beg;
- while ( *end && *end!=',' ) end++;
- char tmp = *end; *end = 0;
- int id = bcf_hdr_id2int(args->hdr_in, BCF_DT_ID, beg);
- beg = tmp ? end + 1 : end;
- if ( is_info && bcf_hdr_idinfo_exists(args->hdr_in,BCF_HL_INFO,id) )
- {
- if ( id >= args->ninfo_tags ) args->ninfo_tags = id + 1;
- hts_expand0(uint8_t, args->ninfo_tags, args->minfo_tags, args->info_tags);
- args->info_tags[id] = 1;
- }
- if ( is_fmt && bcf_hdr_idinfo_exists(args->hdr_in,BCF_HL_FMT,id) )
- {
- if ( id >= args->nfmt_tags ) args->nfmt_tags = id + 1;
- hts_expand0(uint8_t, args->nfmt_tags, args->mfmt_tags, args->fmt_tags);
- args->fmt_tags[id] = 1;
- }
- }
- if ( !args->keep_info && !args->keep_fmt && !args->ninfo_tags && !args->nfmt_tags )
- {
- args->keep_info = args->keep_fmt = 1;
- }
-}
-static void destroy_data(args_t *args)
-{
- free(args->info_tags);
- free(args->fmt_tags);
- if ( args->filter )
- filter_destroy(args->filter);
- int i, nsmpl = bcf_hdr_nsamples(args->hdr_in);
- for (i=0; i<nsmpl; i++)
- {
- if ( args->fh[i] && hts_close(args->fh[i])!=0 ) error("Error: close failed!\n");
- free(args->bnames[i]);
- }
- free(args->bnames);
- free(args->fh);
- bcf_sr_destroy(args->sr);
- bcf_hdr_destroy(args->hdr_out);
- free(args);
-}
-
-static bcf1_t *rec_set_info(args_t *args, bcf1_t *rec)
-{
- bcf1_t *out = bcf_init1();
- out->rid = rec->rid;
- out->pos = rec->pos;
- out->rlen = rec->rlen;
- out->qual = rec->qual;
- out->n_allele = rec->n_allele;
- out->n_sample = 1;
- if ( args->keep_info )
- {
- out->n_info = rec->n_info;
- out->shared.m = out->shared.l = rec->shared.l;
- out->shared.s = (char*) malloc(out->shared.l);
- memcpy(out->shared.s,rec->shared.s,out->shared.l);
- return out;
- }
-
- // build the BCF record
- kstring_t tmp = {0,0,0};
- char *ptr = rec->shared.s;
- kputsn_(ptr, rec->unpack_size[0], &tmp); ptr += rec->unpack_size[0]; // ID
- kputsn_(ptr, rec->unpack_size[1], &tmp); ptr += rec->unpack_size[1]; // REF+ALT
- kputsn_(ptr, rec->unpack_size[2], &tmp); // FILTER
- if ( args->ninfo_tags )
- {
- int i;
- for (i=0; i<rec->n_info; i++)
- {
- bcf_info_t *info = &rec->d.info[i];
- int id = info->key;
- if ( !args->info_tags[id] ) continue;
- kputsn_(info->vptr - info->vptr_off, info->vptr_len + info->vptr_off, &tmp);
- out->n_info++;
- }
- }
- out->shared.m = tmp.m;
- out->shared.s = tmp.s;
- out->shared.l = tmp.l;
- out->unpacked = 0;
- return out;
-}
-
-static bcf1_t *rec_set_format(args_t *args, bcf1_t *src, int ismpl, bcf1_t *dst)
-{
- dst->n_fmt = 0;
- kstring_t tmp = dst->indiv; tmp.l = 0;
- int i;
- for (i=0; i<src->n_fmt; i++)
- {
- bcf_fmt_t *fmt = &src->d.fmt[i];
- int id = fmt->id;
- if ( !args->keep_fmt && !args->fmt_tags[id] ) continue;
-
- bcf_enc_int1(&tmp, id);
- bcf_enc_size(&tmp, fmt->n, fmt->type);
- kputsn_(fmt->p + ismpl*fmt->size, fmt->size, &tmp);
-
- dst->n_fmt++;
- }
- dst->indiv = tmp;
- return dst;
-}
-
-static void process(args_t *args)
-{
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- bcf_unpack(rec, BCF_UN_ALL);
-
- int i, site_pass = 1;
- const uint8_t *smpl_pass = NULL;
- if ( args->filter )
- {
- site_pass = filter_test(args->filter, rec, &smpl_pass);
- if ( args->filter_logic & FLT_EXCLUDE ) site_pass = site_pass ? 0 : 1;
- }
- bcf1_t *out = NULL;
- for (i=0; i<rec->n_sample; i++)
- {
- if ( !args->fh[i] ) continue;
- if ( !smpl_pass && !site_pass ) continue;
- if ( smpl_pass )
- {
- int pass = args->filter_logic & FLT_EXCLUDE ? ( smpl_pass[i] ? 0 : 1) : smpl_pass[i];
- if ( !pass ) continue;
- }
- if ( !out ) out = rec_set_info(args, rec);
- rec_set_format(args, rec, i, out);
- bcf_write(args->fh[i], args->hdr_out, out);
- }
- if ( out ) bcf_destroy(out);
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->output_type = FT_VCF;
- static struct option loptions[] =
- {
- {"keep-tags",required_argument,NULL,'k'},
- {"exclude",required_argument,NULL,'e'},
- {"include",required_argument,NULL,'i'},
- {"regions",required_argument,NULL,'r'},
- {"regions-file",required_argument,NULL,'R'},
- {"samples-file",required_argument,NULL,'S'},
- {"output",required_argument,NULL,'o'},
- {"output-type",required_argument,NULL,'O'},
- {NULL,0,NULL,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "vr:R:t:T:o:O:i:e:k:S:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'k': args->keep_tags = optarg; break;
- case 'e': args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 'T': args->target = optarg; args->target_is_file = 1; break;
- case 't': args->target = optarg; break;
- case 'R': args->region = optarg; args->region_is_file = 1; break;
- case 'S': args->samples_fname = optarg; break;
- case 'r': args->region = optarg; break;
- case 'o': args->output_dir = optarg; break;
- case 'O':
- switch (optarg[0]) {
- case 'b': args->output_type = FT_BCF_GZ; break;
- case 'u': args->output_type = FT_BCF; break;
- case 'z': args->output_type = FT_VCF_GZ; break;
- case 'v': args->output_type = FT_VCF; break;
- default: error("The output type \"%s\" not recognised\n", optarg);
- }
- break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else { error("%s", usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s", usage_text());
- else args->fname = argv[optind];
-
- if ( !args->output_dir ) error("Missing the -o option\n");
- if ( args->filter_logic == (FLT_EXCLUDE|FLT_INCLUDE) ) error("Only one of -i or -e can be given.\n");
-
- init_data(args);
-
- while ( bcf_sr_next_line(args->sr) ) process(args);
-
- destroy_data(args);
- return 0;
-}
-
-
+++ /dev/null
-/* plugins/tag2tag.c -- convert between similar tags
-
- Copyright (C) 2014-2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include "bcftools.h"
-
-
-#define GP_TO_GL 1
-#define GL_TO_PL 2
-#define GP_TO_GT 3
-#define PL_TO_GL 4
-
-static int mode = 0, drop_source_tag = 0;
-static bcf_hdr_t *in_hdr, *out_hdr;
-static float *farr = NULL, thresh = 0.1;
-static int32_t *iarr = NULL;
-static int mfarr = 0, miarr = 0;
-
-const char *about(void)
-{
- return "Convert between similar tags, such as GL, PL and GP.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Convert between similar tags, such as GL, PL and GP.\n"
- "Usage: bcftools +tag2tag [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " --gp-to-gl convert FORMAT/GP to FORMAT/GL\n"
- " --gp-to-gt convert FORMAT/GP to FORMAT/GT by taking argmax of GP\n"
- " --gl-to-pl convert FORMAT/GL to FORMAT/PL\n"
- " --pl-to-gl convert FORMAT/PL to FORMAT/GL\n"
- " -r, --replace drop the source tag\n"
- " -t, --threshold <float> threshold for GP to GT hard-call [0.1]\n"
- "\n"
- "Example:\n"
- " bcftools +tag2tag in.vcf -- -r --gp-to-gl\n"
- "\n";
-}
-
-
-static void init_header(bcf_hdr_t *hdr, const char *ori, int ori_type, const char *new_hdr_line)
-{
- if ( ori )
- bcf_hdr_remove(hdr,ori_type,ori);
-
- bcf_hdr_append(hdr, new_hdr_line);
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- static struct option loptions[] =
- {
- {"replace",no_argument,NULL,'r'},
- {"gp-to-gl",no_argument,NULL,1},
- {"gl-to-pl",no_argument,NULL,2},
- {"gp-to-gt",no_argument,NULL,3},
- {"pl-to-gl",no_argument,NULL,4},
- {"threshold",required_argument,NULL,'t'},
- {NULL,0,NULL,0}
- };
- int c;
- char *src_tag = "GP";
- while ((c = getopt_long(argc, argv, "?hrt:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 1 : src_tag = "GP"; mode = GP_TO_GL; break;
- case 2 : src_tag = "GL"; mode = GL_TO_PL; break;
- case 3 : src_tag = "GP"; mode = GP_TO_GT; break;
- case 4 : src_tag = "PL"; mode = PL_TO_GL; break;
- case 'r': drop_source_tag = 1; break;
- case 't': thresh = atof(optarg); break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( !mode ) mode = GP_TO_GL;
-
- in_hdr = in;
- out_hdr = out;
-
- if ( mode==GP_TO_GL )
- init_header(out_hdr,drop_source_tag?"GP":NULL,BCF_HL_FMT,"##FORMAT=<ID=GL,Number=G,Type=Float,Description=\"Genotype Likelihoods\">");
- else if ( mode==GL_TO_PL )
- init_header(out_hdr,drop_source_tag?"GL":NULL,BCF_HL_FMT,"##FORMAT=<ID=PL,Number=G,Type=Integer,Description=\"Phred-scaled genotype likelihoods\">");
- else if ( mode==PL_TO_GL )
- init_header(out_hdr,drop_source_tag?"PL":NULL,BCF_HL_FMT,"##FORMAT=<ID=GL,Number=G,Type=Float,Description=\"Genotype likelihoods\">");
- else if ( mode==GP_TO_GT ) {
- if (thresh<0||thresh>1) error("--threshold must be in the range [0,1]: %f\n", thresh);
- init_header(out_hdr,drop_source_tag?"GP":NULL,BCF_HL_FMT,"##FORMAT=<ID=GT,Number=1,Type=String,Description=\"Genotype\">");
- }
-
- int tag_id;
- if ( (tag_id=bcf_hdr_id2int(in_hdr,BCF_DT_ID,src_tag))<0 || !bcf_hdr_idinfo_exists(in_hdr,BCF_HL_FMT,tag_id) )
- error("The source tag does not exist: %s\n", src_tag);
-
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int i, n;
- if ( mode==GP_TO_GL )
- {
- n = bcf_get_format_float(in_hdr,rec,"GP",&farr,&mfarr);
- if ( n<=0 ) return rec;
- for (i=0; i<n; i++)
- {
- if ( bcf_float_is_missing(farr[i]) || bcf_float_is_vector_end(farr[i]) ) continue;
- farr[i] = farr[i] ? log10(farr[i]) : -99;
- }
- bcf_update_format_float(out_hdr,rec,"GL",farr,n);
- if ( drop_source_tag )
- bcf_update_format_float(out_hdr,rec,"GP",NULL,0);
- }
- else if ( mode==PL_TO_GL )
- {
- n = bcf_get_format_int32(in_hdr,rec,"PL",&iarr,&miarr);
- if ( n<=0 ) return rec;
- hts_expand(float, n, mfarr, farr);
- for (i=0; i<n; i++)
- {
- if ( iarr[i]==bcf_int32_missing )
- bcf_float_set_missing(farr[i]);
- else if ( iarr[i]==bcf_int32_vector_end )
- bcf_float_set_vector_end(farr[i]);
- else
- farr[i] = -0.1 * iarr[i];
- }
- bcf_update_format_float(out_hdr,rec,"GL",farr,n);
- if ( drop_source_tag )
- bcf_update_format_int32(out_hdr,rec,"PL",NULL,0);
- }
- else if ( mode==GL_TO_PL )
- {
- n = bcf_get_format_float(in_hdr,rec,"GL",&farr,&mfarr);
- if ( n<=0 ) return rec;
- hts_expand(int32_t, n, miarr, iarr);
- for (i=0; i<n; i++)
- {
- if ( bcf_float_is_missing(farr[i]) )
- iarr[i] = bcf_int32_missing;
- else if ( bcf_float_is_vector_end(farr[i]) )
- iarr[i] = bcf_int32_vector_end;
- else
- iarr[i] = lroundf(-10 * farr[i]);
- }
- bcf_update_format_int32(out_hdr,rec,"PL",iarr,n);
- if ( drop_source_tag )
- bcf_update_format_float(out_hdr,rec,"GL",NULL,0);
- }
- else if ( mode==GP_TO_GT )
- {
- int nals = rec->n_allele;
- int nsmpl = bcf_hdr_nsamples(in_hdr);
- hts_expand(int32_t,nsmpl*2,miarr,iarr);
-
- n = bcf_get_format_float(in_hdr,rec,"GP",&farr,&mfarr);
- if ( n<=0 ) return rec;
-
- n /= nsmpl;
- for (i=0; i<nsmpl; i++)
- {
- float *ptr = farr + i*n;
- if ( bcf_float_is_missing(ptr[0]) )
- {
- iarr[2*i] = iarr[2*i+1] = bcf_gt_missing;
- continue;
- }
-
- int j, jmax = 0;
- for (j=1; j<n; j++)
- {
- if ( bcf_float_is_missing(ptr[j]) || bcf_float_is_vector_end(ptr[j]) ) break;
- if ( ptr[j] > ptr[jmax] ) jmax = j;
- }
-
- // haploid genotype
- if ( j==nals )
- {
- iarr[2*i] = ptr[jmax] < 1-thresh ? bcf_gt_missing : bcf_gt_unphased(jmax);
- iarr[2*i+1] = bcf_int32_vector_end;
- continue;
- }
-
- if ( j!=nals*(nals+1)/2 )
- error("Wrong number of GP values for diploid genotype at %s:%d, expected %d, found %d\n",
- bcf_seqname(in_hdr,rec),rec->pos+1, nals*(nals+1)/2,j);
-
- if (ptr[jmax] < 1-thresh)
- {
- iarr[2*i] = iarr[2*i+1] = bcf_gt_missing;
- continue;
- }
-
- // most common case: RR
- if ( jmax==0 )
- {
- iarr[2*i] = iarr[2*i+1] = bcf_gt_unphased(0);
- continue;
- }
-
- int a,b;
- bcf_gt2alleles(jmax,&a,&b);
- iarr[2*i] = bcf_gt_unphased(a);
- iarr[2*i+1] = bcf_gt_unphased(b);
- }
- bcf_update_genotypes(out_hdr,rec,iarr,nsmpl*2);
- if ( drop_source_tag )
- bcf_update_format_float(out_hdr,rec,"GP",NULL,0);
- }
- return rec;
-}
-
-void destroy(void)
-{
- free(farr);
- free(iarr);
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* plugins/tag2tag.c -- convert between similar tags
-
- Copyright (C) 2014-2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include "bcftools.h"
-
-
-#define GP_TO_GL 1
-#define GL_TO_PL 2
-#define GP_TO_GT 3
-#define PL_TO_GL 4
-
-static int mode = 0, drop_source_tag = 0;
-static bcf_hdr_t *in_hdr, *out_hdr;
-static float *farr = NULL, thresh = 0.1;
-static int32_t *iarr = NULL;
-static int mfarr = 0, miarr = 0;
-
-const char *about(void)
-{
- return "Convert between similar tags, such as GL, PL and GP.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Convert between similar tags, such as GL, PL and GP.\n"
- "Usage: bcftools +tag2tag [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " --gp-to-gl convert FORMAT/GP to FORMAT/GL\n"
- " --gp-to-gt convert FORMAT/GP to FORMAT/GT by taking argmax of GP\n"
- " --gl-to-pl convert FORMAT/GL to FORMAT/PL\n"
- " --pl-to-gl convert FORMAT/PL to FORMAT/GL\n"
- " -r, --replace drop the source tag\n"
- " -t, --threshold <float> threshold for GP to GT hard-call [0.1]\n"
- "\n"
- "Example:\n"
- " bcftools +tag2tag in.vcf -- -r --gp-to-gl\n"
- "\n";
-}
-
-
-static void init_header(bcf_hdr_t *hdr, const char *ori, int ori_type, const char *new_hdr_line)
-{
- if ( ori )
- bcf_hdr_remove(hdr,ori_type,ori);
-
- bcf_hdr_append(hdr, new_hdr_line);
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- static struct option loptions[] =
- {
- {"replace",no_argument,NULL,'r'},
- {"gp-to-gl",no_argument,NULL,1},
- {"gl-to-pl",no_argument,NULL,2},
- {"gp-to-gt",no_argument,NULL,3},
- {"pl-to-gl",no_argument,NULL,4},
- {"threshold",required_argument,NULL,'t'},
- {NULL,0,NULL,0}
- };
- int c;
- char *src_tag = "GP";
- while ((c = getopt_long(argc, argv, "?hrt:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 1 : src_tag = "GP"; mode = GP_TO_GL; break;
- case 2 : src_tag = "GL"; mode = GL_TO_PL; break;
- case 3 : src_tag = "GP"; mode = GP_TO_GT; break;
- case 4 : src_tag = "PL"; mode = PL_TO_GL; break;
- case 'r': drop_source_tag = 1; break;
- case 't': thresh = atof(optarg); break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( !mode ) mode = GP_TO_GL;
-
- in_hdr = in;
- out_hdr = out;
-
- if ( mode==GP_TO_GL )
- init_header(out_hdr,drop_source_tag?"GP":NULL,BCF_HL_FMT,"##FORMAT=<ID=GL,Number=G,Type=Float,Description=\"Genotype Likelihoods\">");
- else if ( mode==GL_TO_PL )
- init_header(out_hdr,drop_source_tag?"GL":NULL,BCF_HL_FMT,"##FORMAT=<ID=PL,Number=G,Type=Integer,Description=\"Phred-scaled genotype likelihoods\">");
- else if ( mode==PL_TO_GL )
- init_header(out_hdr,drop_source_tag?"PL":NULL,BCF_HL_FMT,"##FORMAT=<ID=GL,Number=G,Type=Float,Description=\"Genotype likelihoods\">");
- else if ( mode==GP_TO_GT ) {
- if (thresh<0||thresh>1) error("--threshold must be in the range [0,1]: %f\n", thresh);
- init_header(out_hdr,drop_source_tag?"GP":NULL,BCF_HL_FMT,"##FORMAT=<ID=GT,Number=1,Type=String,Description=\"Genotype\">");
- }
-
- int tag_id;
- if ( (tag_id=bcf_hdr_id2int(in_hdr,BCF_DT_ID,src_tag))<0 || !bcf_hdr_idinfo_exists(in_hdr,BCF_HL_FMT,tag_id) )
- error("The source tag does not exist: %s\n", src_tag);
-
- return 0;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int i, n;
- if ( mode==GP_TO_GL )
- {
- n = bcf_get_format_float(in_hdr,rec,"GP",&farr,&mfarr);
- if ( n<=0 ) return rec;
- for (i=0; i<n; i++)
- {
- if ( bcf_float_is_missing(farr[i]) || bcf_float_is_vector_end(farr[i]) ) continue;
- farr[i] = farr[i] ? log10(farr[i]) : -99;
- }
- bcf_update_format_float(out_hdr,rec,"GL",farr,n);
- if ( drop_source_tag )
- bcf_update_format_float(out_hdr,rec,"GP",NULL,0);
- }
- else if ( mode==PL_TO_GL )
- {
- n = bcf_get_format_int32(in_hdr,rec,"PL",&iarr,&miarr);
- if ( n<=0 ) return rec;
- hts_expand(float, n, mfarr, farr);
- for (i=0; i<n; i++)
- {
- if ( iarr[i]==bcf_int32_missing )
- bcf_float_set_missing(farr[i]);
- else if ( iarr[i]==bcf_int32_vector_end )
- bcf_float_set_vector_end(farr[i]);
- else
- farr[i] = -0.1 * iarr[i];
- }
- bcf_update_format_float(out_hdr,rec,"GL",farr,n);
- if ( drop_source_tag )
- bcf_update_format_int32(out_hdr,rec,"PL",NULL,0);
- }
- else if ( mode==GL_TO_PL )
- {
- n = bcf_get_format_float(in_hdr,rec,"GL",&farr,&mfarr);
- if ( n<=0 ) return rec;
- hts_expand(int32_t, n, miarr, iarr);
- for (i=0; i<n; i++)
- {
- if ( bcf_float_is_missing(farr[i]) )
- iarr[i] = bcf_int32_missing;
- else if ( bcf_float_is_vector_end(farr[i]) )
- iarr[i] = bcf_int32_vector_end;
- else
- iarr[i] = lroundf(-10 * farr[i]);
- }
- bcf_update_format_int32(out_hdr,rec,"PL",iarr,n);
- if ( drop_source_tag )
- bcf_update_format_float(out_hdr,rec,"GL",NULL,0);
- }
- else if ( mode==GP_TO_GT )
- {
- int nals = rec->n_allele;
- int nsmpl = bcf_hdr_nsamples(in_hdr);
- hts_expand(int32_t,nsmpl*2,miarr,iarr);
-
- n = bcf_get_format_float(in_hdr,rec,"GP",&farr,&mfarr);
- if ( n<=0 ) return rec;
-
- n /= nsmpl;
- for (i=0; i<nsmpl; i++)
- {
- float *ptr = farr + i*n;
- if ( bcf_float_is_missing(ptr[0]) )
- {
- iarr[2*i] = iarr[2*i+1] = bcf_gt_missing;
- continue;
- }
-
- int j, jmax = 0;
- for (j=1; j<n; j++)
- {
- if ( bcf_float_is_missing(ptr[j]) || bcf_float_is_vector_end(ptr[j]) ) break;
- if ( ptr[j] > ptr[jmax] ) jmax = j;
- }
-
- // haploid genotype
- if ( j==nals )
- {
- iarr[2*i] = ptr[jmax] < 1-thresh ? bcf_gt_missing : bcf_gt_unphased(jmax);
- iarr[2*i+1] = bcf_int32_vector_end;
- continue;
- }
-
- if ( j!=nals*(nals+1)/2 )
- error("Wrong number of GP values for diploid genotype at %s:%d, expected %d, found %d\n",
- bcf_seqname(in_hdr,rec),rec->pos+1, nals*(nals+1)/2,j);
-
- if (ptr[jmax] < 1-thresh)
- {
- iarr[2*i] = iarr[2*i+1] = bcf_gt_missing;
- continue;
- }
-
- // most common case: RR
- if ( jmax==0 )
- {
- iarr[2*i] = iarr[2*i+1] = bcf_gt_unphased(0);
- continue;
- }
-
- int a,b;
- bcf_gt2alleles(jmax,&a,&b);
- iarr[2*i] = bcf_gt_unphased(a);
- iarr[2*i+1] = bcf_gt_unphased(b);
- }
- bcf_update_genotypes(out_hdr,rec,iarr,nsmpl*2);
- if ( drop_source_tag )
- bcf_update_format_float(out_hdr,rec,"GP",NULL,0);
- }
- return rec;
-}
-
-void destroy(void)
-{
- free(farr);
- free(iarr);
-}
-
-
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2018 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <unistd.h> // for isatty
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include "bcftools.h"
-#include "filter.h"
-
-
-// Logic of the filters: include or exclude sites which match the filters?
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-#define iCHILD 0
-#define iFATHER 1
-#define iMOTHER 2
-
-typedef struct
-{
- int idx[3]; // VCF sample index for father, mother and child
- int pass; // do all three pass the filters?
-}
-trio_t;
-
-typedef struct
-{
- uint32_t
- npass, // number of genotypes passing the filter
- nnon_ref, // number of non-reference genotypes
- nmendel_err, // number of mendelian errors
- nnovel, // a singleton allele, but observed only in the child. Counted as mendel_err as well.
- nsingleton, // het mother or father different from everyone else
- ndoubleton, // het mother+child or father+child different from everyone else
- nts, ntv; // number of transitions and transversions
-}
-trio_stats_t;
-
-typedef struct
-{
- trio_stats_t *stats;
- filter_t *filter;
- char *expr;
-}
-flt_stats_t;
-
-typedef struct
-{
- int argc, filter_logic, regions_is_file, targets_is_file;
- int nflt_str;
- char *filter_str, **flt_str;
- char **argv, *ped_fname, *output_fname, *fname, *regions, *targets;
- bcf_srs_t *sr;
- bcf_hdr_t *hdr;
- trio_t *trio;
- int ntrio, mtrio;
- flt_stats_t *filters;
- int nfilters;
- int32_t *gt_arr, *ac, *ac_trio;
- int mgt_arr, mac, mac_trio;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Calculate transmission rate and other stats in trio children.\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Calculate transmission rate in trio children. Use curly brackets to scan\n"
- " a range of values simultaneously\n"
- "Usage: bcftools +trio-stats [Plugin Options]\n"
- "Plugin options:\n"
- " -e, --exclude EXPR exclude sites and samples for which the expression is true\n"
- " -i, --include EXPR include sites and samples for which the expression is true\n"
- " -o, --output FILE output file name [stdout]\n"
- " -p, --ped FILE PED file\n"
- " -r, --regions REG restrict to comma-separated list of regions\n"
- " -R, --regions-file FILE restrict to regions listed in a file\n"
- " -t, --targets REG similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file FILE similar to -R but streams rather than index-jumps\n"
- "\n"
- "Example:\n"
- " bcftools +trio-stats -p file.ped -i 'GQ>{10,20,30,40,50}' file.bcf\n"
- "\n";
-}
-
-static int cmp_trios(const void *_a, const void *_b)
-{
- trio_t *a = (trio_t *) _a;
- trio_t *b = (trio_t *) _b;
- int i;
- int amin = a->idx[0];
- for (i=1; i<3; i++)
- if ( amin > a->idx[i] ) amin = a->idx[i];
- int bmin = b->idx[0];
- for (i=1; i<3; i++)
- if ( bmin > b->idx[i] ) bmin = b->idx[i];
- if ( amin < bmin ) return -1;
- if ( amin > bmin ) return 1;
- return 0;
-}
-
-static void parse_ped(args_t *args, char *fname)
-{
- htsFile *fp = hts_open(fname, "r");
- if ( !fp ) error("Could not read: %s\n", fname);
-
- kstring_t str = {0,0,0};
- if ( hts_getline(fp, KS_SEP_LINE, &str) <= 0 ) error("Empty file: %s\n", fname);
-
- int moff = 0, *off = NULL;
- do
- {
- // familyID sampleID paternalID maternalID sex phenotype population relationship siblings secondOrder thirdOrder children comment
- // BB03 HG01884 HG01885 HG01956 2 0 ACB child 0 0 0 0
- int ncols = ksplit_core(str.s,0,&moff,&off);
- if ( ncols<4 ) error("Could not parse the ped file: %s\n", str.s);
-
- int father = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[2]]);
- if ( father<0 ) continue;
- int mother = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[3]]);
- if ( mother<0 ) continue;
- int child = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[1]]);
- if ( child<0 ) continue;
-
- args->ntrio++;
- hts_expand0(trio_t,args->ntrio,args->mtrio,args->trio);
- trio_t *trio = &args->trio[args->ntrio-1];
- trio->idx[iFATHER] = father;
- trio->idx[iMOTHER] = mother;
- trio->idx[iCHILD] = child;
- }
- while ( hts_getline(fp, KS_SEP_LINE, &str)>=0 );
-
- fprintf(stderr,"Identified %d complete trios in the VCF file\n", args->ntrio);
-
- // sort the sample by index so that they are accessed more or less sequentially
- qsort(args->trio,args->ntrio,sizeof(trio_t),cmp_trios);
-
- free(str.s);
- free(off);
- hts_close(fp);
-}
-
-static void parse_filters(args_t *args)
-{
- if ( !args->filter_str ) return;
- int mflt = 1;
- args->nflt_str = 1;
- args->flt_str = (char**) malloc(sizeof(char*));
- args->flt_str[0] = strdup(args->filter_str);
- while (1)
- {
- int i, expanded = 0;
- for (i=args->nflt_str-1; i>=0; i--)
- {
- char *exp_beg = strchr(args->flt_str[i], '{');
- if ( !exp_beg ) continue;
- char *exp_end = strchr(exp_beg+1, '}');
- if ( !exp_end ) error("Could not parse the expression: %s\n", args->filter_str);
- char *beg = exp_beg+1, *mid = beg;
- while ( mid<exp_end )
- {
- while ( mid<exp_end && *mid!=',' ) mid++;
- kstring_t tmp = {0,0,0};
- kputsn(args->flt_str[i], exp_beg - args->flt_str[i], &tmp);
- kputsn(beg, mid - beg, &tmp);
- kputs(exp_end+1, &tmp);
- args->nflt_str++;
- hts_expand(char*, args->nflt_str, mflt, args->flt_str);
- args->flt_str[args->nflt_str-1] = tmp.s;
- beg = ++mid;
- }
- expanded = 1;
- free(args->flt_str[i]);
- memmove(&args->flt_str[i], &args->flt_str[i+1], (args->nflt_str-i-1)*sizeof(*args->flt_str));
- args->nflt_str--;
- args->flt_str[args->nflt_str] = NULL;
- }
- if ( !expanded ) break;
- }
-
- fprintf(stderr,"Collecting data for %d filtering expressions\n", args->nflt_str);
-}
-
-static void init_data(args_t *args)
-{
- args->sr = bcf_sr_init();
- if ( args->regions )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, args->regions, args->regions_is_file)<0 ) error("Failed to read the regions: %s\n",args->regions);
- }
- if ( args->targets && bcf_sr_set_targets(args->sr, args->targets, args->targets_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->targets);
- if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr = bcf_sr_get_header(args->sr,0);
-
- parse_ped(args, args->ped_fname);
- parse_filters(args);
-
- int i;
- if ( !args->nflt_str )
- {
- args->filters = (flt_stats_t*) calloc(1, sizeof(flt_stats_t));
- args->nfilters = 1;
- args->filters[0].expr = strdup("all");
- }
- else
- {
- args->nfilters = args->nflt_str;
- args->filters = (flt_stats_t*) calloc(args->nfilters, sizeof(flt_stats_t));
- for (i=0; i<args->nfilters; i++)
- {
- args->filters[i].filter = filter_init(args->hdr, args->flt_str[i]);
- args->filters[i].expr = strdup(args->flt_str[i]);
-
- // replace tab's with spaces so that the output stays parsable
- char *tmp = args->filters[i].expr;
- while ( *tmp )
- {
- if ( *tmp=='\t' ) *tmp = ' ';
- tmp++;
- }
- }
- }
- for (i=0; i<args->nfilters; i++)
- args->filters[i].stats = (trio_stats_t*) calloc(args->ntrio,sizeof(trio_stats_t));
-}
-static void destroy_data(args_t *args)
-{
- int i;
- for (i=0; i<args->nfilters; i++)
- {
- if ( args->filters[i].filter ) filter_destroy(args->filters[i].filter);
- free(args->filters[i].stats);
- free(args->filters[i].expr);
- }
- free(args->filters);
- for (i=0; i<args->nflt_str; i++) free(args->flt_str[i]);
- free(args->flt_str);
- bcf_sr_destroy(args->sr);
- free(args->trio);
- free(args->ac);
- free(args->ac_trio);
- free(args->gt_arr);
- free(args);
-}
-static void report_stats(args_t *args)
-{
- int i = 0,j;
- FILE *fh = !args->output_fname || !strcmp("-",args->output_fname) ? stdout : fopen(args->output_fname,"w");
- if ( !fh ) error("Could not open the file for writing: %s\n", args->output_fname);
- fprintf(fh,"# CMD line shows the command line used to generate this output\n");
- fprintf(fh,"# DEF lines define expressions for all tested thresholds\n");
- fprintf(fh,"# FLT* lines report numbers for every threshold and every trio:\n");
- fprintf(fh,"# %d) filter id\n", ++i);
- fprintf(fh,"# %d) child\n", ++i);
- fprintf(fh,"# %d) father\n", ++i);
- fprintf(fh,"# %d) mother\n", ++i);
- fprintf(fh,"# %d) number of valid trio genotypes (all trio members pass filters, all non-missing)\n", ++i);
- fprintf(fh,"# %d) number of non-reference trio GTs (at least one trio member carries an alternate allele)\n", ++i);
- fprintf(fh,"# %d) number of Mendelian errors\n", ++i);
- fprintf(fh,"# %d) number of novel singleton alleles in the child (counted also as a Mendelian error)\n", ++i);
- fprintf(fh,"# %d) number of untransmitted singletons, present only in one parent\n", ++i);
- fprintf(fh,"# %d) number of transmitted singletons, present only in one parent and the child\n", ++i);
- fprintf(fh,"# %d) number of transitions, all ALT alleles present in the trio are considered\n", ++i);
- fprintf(fh,"# %d) number of transversions, all ALT alleles present in the trio are considered\n", ++i);
- fprintf(fh,"# %d) overall ts/tv, all ALT alleles present in the trio are considered\n", ++i);
- fprintf(fh, "CMD\t%s", args->argv[0]);
- for (i=1; i<args->argc; i++) fprintf(fh, " %s",args->argv[i]);
- fprintf(fh, "\n");
- for (i=0; i<args->nfilters; i++)
- {
- flt_stats_t *flt = &args->filters[i];
- fprintf(fh,"DEF\tFLT%d\t%s\n", i, flt->expr);
- }
- for (i=0; i<args->nfilters; i++)
- {
- flt_stats_t *flt = &args->filters[i];
- for (j=0; j<args->ntrio; j++)
- {
- fprintf(fh,"FLT%d", i);
- fprintf(fh,"\t%s",args->hdr->samples[args->trio[j].idx[iCHILD]]);
- fprintf(fh,"\t%s",args->hdr->samples[args->trio[j].idx[iFATHER]]);
- fprintf(fh,"\t%s",args->hdr->samples[args->trio[j].idx[iMOTHER]]);
- trio_stats_t *stats = &flt->stats[j];
- fprintf(fh,"\t%d", stats->npass);
- fprintf(fh,"\t%d", stats->nnon_ref);
- fprintf(fh,"\t%d", stats->nmendel_err);
- fprintf(fh,"\t%d", stats->nnovel);
- fprintf(fh,"\t%d", stats->nsingleton);
- fprintf(fh,"\t%d", stats->ndoubleton);
- fprintf(fh,"\t%d", stats->nts);
- fprintf(fh,"\t%d", stats->ntv);
- fprintf(fh,"\t%.2f", stats->ntv ? (float)stats->nts/stats->ntv : INFINITY);
- fprintf(fh,"\n");
- }
- }
- if ( fclose(fh)!=0 ) error("Close failed: %s\n", (!args->output_fname || !strcmp("-",args->output_fname)) ? "stdout" : args->output_fname);
-}
-
-static inline int parse_genotype(int32_t *arr, int ngt1, int idx, int als[2])
-{
- int32_t *ptr = arr + ngt1 * idx;
- if ( bcf_gt_is_missing(ptr[0]) ) return -1;
- als[0] = bcf_gt_allele(ptr[0]);
-
- // treat haploid GTs as homozygous diploid
- if ( ngt1==1 || ptr[1]==bcf_int32_vector_end ) { als[1] = als[0]; return 0; }
-
- if ( bcf_gt_is_missing(ptr[1]) ) return -1;
- als[1] = bcf_gt_allele(ptr[1]);
-
- return 0;
-}
-
-static void process_record(args_t *args, bcf1_t *rec, flt_stats_t *flt)
-{
- int i,j;
-
- // Find out which trios pass and if the site passes
- if ( flt->filter )
- {
- uint8_t *smpl_pass = NULL;
- int pass_site = filter_test(flt->filter, rec, (const uint8_t**) &smpl_pass);
- if ( args->filter_logic & FLT_EXCLUDE )
- {
- if ( pass_site )
- {
- if ( !smpl_pass ) return;
- pass_site = 0;
- for (i=0; i<args->ntrio; i++)
- {
- int pass_trio = 1;
- for (j=0; j<3; j++)
- {
- int idx = args->trio[i].idx[j];
- if ( smpl_pass[idx] ) { pass_trio = 0; break; }
- }
- args->trio[i].pass = pass_trio;
- if ( pass_trio ) pass_site = 1;
- }
- if ( !pass_site ) return;
- }
- else
- for (i=0; i<args->ntrio; i++) args->trio[i].pass = 1;
- }
- else if ( !pass_site ) return;
- else if ( smpl_pass )
- {
- pass_site = 0;
- for (i=0; i<args->ntrio; i++)
- {
- int pass_trio = 1;
- for (j=0; j<3; j++)
- {
- int idx = args->trio[i].idx[j];
- if ( !smpl_pass[idx] ) { pass_trio = 0; break; }
- }
- args->trio[i].pass = pass_trio;
- if ( pass_trio ) pass_site = 1;
- }
- if ( !pass_site ) return;
- }
- else
- for (i=0; i<args->ntrio; i++) args->trio[i].pass = 1;
- }
-
- // Find out the allele counts. Try to use INFO/AC, if not present, determine from the genotypes
- hts_expand(int, rec->n_allele, args->mac, args->ac);
- if ( !bcf_calc_ac(args->hdr, rec, args->ac, BCF_UN_INFO|BCF_UN_FMT) ) return;
- hts_expand(int, rec->n_allele, args->mac_trio, args->ac_trio);
-
- // Get the genotypes
- int ngt = bcf_get_genotypes(args->hdr, rec, &args->gt_arr, &args->mgt_arr);
- if ( ngt<0 ) return;
- int ngt1 = ngt / rec->n_sample;
-
-
- // For ts/tv: numeric code of the reference allele, -1 for insertions
- int ref = !rec->d.allele[0][1] ? bcf_acgt2int(*rec->d.allele[0]) : -1;
-
- int star_allele = -1;
- for (i=1; i<rec->n_allele; i++)
- if ( !rec->d.allele[i][1] && rec->d.allele[i][0]=='*' ) { star_allele = i; break; }
-
- // Run the stats
- for (i=0; i<args->ntrio; i++)
- {
- if ( flt->filter && !args->trio[i].pass ) continue;
- trio_stats_t *stats = &flt->stats[i];
-
- // Determine the alternate allele and the genotypes, skip if any of the alleles is missing.
- // the order is: child, father, mother
- int als[6], *als_child = als, *als_father = als+2, *als_mother = als+4;
- if ( parse_genotype(args->gt_arr, ngt1, args->trio[i].idx[iCHILD], als_child) < 0 ) continue;
- if ( parse_genotype(args->gt_arr, ngt1, args->trio[i].idx[iFATHER], als_father) < 0 ) continue;
- if ( parse_genotype(args->gt_arr, ngt1, args->trio[i].idx[iMOTHER], als_mother) < 0 ) continue;
-
- stats->npass++;
-
- // Has the trio an alternate allele other than *?
- int has_star_allele = 0, has_nonref = 0;
- memset(args->ac_trio,0,rec->n_allele*sizeof(*args->ac_trio));
- for (j=0; j<6; j++)
- {
- if ( als[j]==star_allele ) { has_star_allele = 1; continue; }
- if ( als[j]==0 ) continue;
- has_nonref = 1;
- args->ac_trio[ als[j] ]++;
- }
- if ( !has_nonref ) continue; // only ref or * in this trio
-
- stats->nnon_ref++;
-
- // Calculate ts/tv. It does the right thing and handles also HetAA genotypes
- if ( ref != -1 )
- {
- int has_ts = 0, has_tv = 0;
- for (j=0; j<6; j++)
- {
- if ( als[j]==0 || als[j]==star_allele ) continue;
- if ( als[j] >= rec->n_allele )
- error("The GT index is out of range at %s:%d in %s\n", bcf_seqname(args->hdr,rec),rec->pos+1,args->hdr->samples[args->trio[i].idx[j/2]]);
- if ( rec->d.allele[als[j]][1] ) continue;
-
- int alt = bcf_acgt2int(rec->d.allele[als[j]][0]);
- if ( abs(ref-alt)==2 ) has_ts = 1;
- else has_tv = 1;
- }
- if ( has_ts ) stats->nts++;
- if ( has_tv ) stats->ntv++;
- }
-
- // Skip some stats if the star allele is present as it was already checked at the primary record, we do not want to count the same
- // thing multiple times. There can be other alternate allele, but we ignore that for simplicity.
- if ( has_star_allele ) continue;
-
- // Detect mendelian errors
- int mendel_ok = (als_child[0]==als_father[0] || als_child[0]==als_father[1]) && (als_child[1]==als_mother[0] || als_child[1]==als_mother[1]) ? 1 : 0;
- if ( !mendel_ok ) mendel_ok = (als_child[1]==als_father[0] || als_child[1]==als_father[1]) && (als_child[0]==als_mother[0] || als_child[0]==als_mother[1]) ? 1 : 0;
- if ( !mendel_ok ) stats->nmendel_err++;
-
- // Is this a singleton, doubleton, neither?
- for (j=1; j<rec->n_allele; j++)
- {
- if ( args->ac_trio[j]==1 && args->ac[j]==1 ) // singleton (in parent) or novel (in child)
- {
- if ( als_child[0]==j || als_child[1]==j ) stats->nnovel++;
- else stats->nsingleton++;
- }
- else if ( args->ac_trio[j]==2 && args->ac[j]==2 ) // possibly a doubleton
- {
- if ( (als_child[0]==j || als_child[1]==j) && (als_child[0]!=j || als_child[1]!=j) ) stats->ndoubleton++;
- }
- }
- }
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->output_fname = "-";
- static struct option loptions[] =
- {
- {"include",required_argument,0,'i'},
- {"exclude",required_argument,0,'e'},
- {"output",required_argument,NULL,'o'},
- {"ped",required_argument,NULL,'p'},
- {"regions",1,0,'r'},
- {"regions-file",1,0,'R'},
- {"targets",1,0,'t'},
- {"targets-file",1,0,'T'},
- {NULL,0,NULL,0}
- };
- int c, i;
- while ((c = getopt_long(argc, argv, "p:o:s:i:e:r:R:t:T:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'e': args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 't': args->targets = optarg; break;
- case 'T': args->targets = optarg; args->targets_is_file = 1; break;
- case 'r': args->regions = optarg; break;
- case 'R': args->regions = optarg; args->regions_is_file = 1; break;
- case 'o': args->output_fname = optarg; break;
- case 'p': args->ped_fname = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else { error("%s", usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s", usage_text());
- else args->fname = argv[optind];
-
- if ( !args->ped_fname ) error("Missing the -p, --ped option\n");
-
- init_data(args);
-
- while ( bcf_sr_next_line(args->sr) )
- {
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- for (i=0; i<args->nfilters; i++)
- process_record(args, rec, &args->filters[i]);
- }
-
- report_stats(args);
- destroy_data(args);
-
- return 0;
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2018 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <unistd.h> // for isatty
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/synced_bcf_reader.h>
-#include <htslib/vcfutils.h>
-#include "bcftools.h"
-#include "filter.h"
-
-
-// Logic of the filters: include or exclude sites which match the filters?
-#define FLT_INCLUDE 1
-#define FLT_EXCLUDE 2
-
-#define iCHILD 0
-#define iFATHER 1
-#define iMOTHER 2
-
-typedef struct
-{
- int idx[3]; // VCF sample index for father, mother and child
- int pass; // do all three pass the filters?
-}
-trio_t;
-
-typedef struct
-{
- uint32_t
- npass, // number of genotypes passing the filter
- nnon_ref, // number of non-reference genotypes
- nmendel_err, // number of mendelian errors
- nnovel, // a singleton allele, but observed only in the child. Counted as mendel_err as well.
- nsingleton, // het mother or father different from everyone else
- ndoubleton, // het mother+child or father+child different from everyone else
- nts, ntv; // number of transitions and transversions
-}
-trio_stats_t;
-
-typedef struct
-{
- trio_stats_t *stats;
- filter_t *filter;
- char *expr;
-}
-flt_stats_t;
-
-typedef struct
-{
- int argc, filter_logic, regions_is_file, targets_is_file;
- int nflt_str;
- char *filter_str, **flt_str;
- char **argv, *ped_fname, *output_fname, *fname, *regions, *targets;
- bcf_srs_t *sr;
- bcf_hdr_t *hdr;
- trio_t *trio;
- int ntrio, mtrio;
- flt_stats_t *filters;
- int nfilters;
- int32_t *gt_arr, *ac, *ac_trio;
- int mgt_arr, mac, mac_trio;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Calculate transmission rate and other stats in trio children.\n";
-}
-
-static const char *usage_text(void)
-{
- return
- "\n"
- "About: Calculate transmission rate in trio children. Use curly brackets to scan\n"
- " a range of values simultaneously\n"
- "Usage: bcftools +trio-stats [Plugin Options]\n"
- "Plugin options:\n"
- " -e, --exclude EXPR exclude sites and samples for which the expression is true\n"
- " -i, --include EXPR include sites and samples for which the expression is true\n"
- " -o, --output FILE output file name [bcftools_stdout]\n"
- " -p, --ped FILE PED file\n"
- " -r, --regions REG restrict to comma-separated list of regions\n"
- " -R, --regions-file FILE restrict to regions listed in a file\n"
- " -t, --targets REG similar to -r but streams rather than index-jumps\n"
- " -T, --targets-file FILE similar to -R but streams rather than index-jumps\n"
- "\n"
- "Example:\n"
- " bcftools +trio-stats -p file.ped -i 'GQ>{10,20,30,40,50}' file.bcf\n"
- "\n";
-}
-
-static int cmp_trios(const void *_a, const void *_b)
-{
- trio_t *a = (trio_t *) _a;
- trio_t *b = (trio_t *) _b;
- int i;
- int amin = a->idx[0];
- for (i=1; i<3; i++)
- if ( amin > a->idx[i] ) amin = a->idx[i];
- int bmin = b->idx[0];
- for (i=1; i<3; i++)
- if ( bmin > b->idx[i] ) bmin = b->idx[i];
- if ( amin < bmin ) return -1;
- if ( amin > bmin ) return 1;
- return 0;
-}
-
-static void parse_ped(args_t *args, char *fname)
-{
- htsFile *fp = hts_open(fname, "r");
- if ( !fp ) error("Could not read: %s\n", fname);
-
- kstring_t str = {0,0,0};
- if ( hts_getline(fp, KS_SEP_LINE, &str) <= 0 ) error("Empty file: %s\n", fname);
-
- int moff = 0, *off = NULL;
- do
- {
- // familyID sampleID paternalID maternalID sex phenotype population relationship siblings secondOrder thirdOrder children comment
- // BB03 HG01884 HG01885 HG01956 2 0 ACB child 0 0 0 0
- int ncols = ksplit_core(str.s,0,&moff,&off);
- if ( ncols<4 ) error("Could not parse the ped file: %s\n", str.s);
-
- int father = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[2]]);
- if ( father<0 ) continue;
- int mother = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[3]]);
- if ( mother<0 ) continue;
- int child = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[1]]);
- if ( child<0 ) continue;
-
- args->ntrio++;
- hts_expand0(trio_t,args->ntrio,args->mtrio,args->trio);
- trio_t *trio = &args->trio[args->ntrio-1];
- trio->idx[iFATHER] = father;
- trio->idx[iMOTHER] = mother;
- trio->idx[iCHILD] = child;
- }
- while ( hts_getline(fp, KS_SEP_LINE, &str)>=0 );
-
- fprintf(bcftools_stderr,"Identified %d complete trios in the VCF file\n", args->ntrio);
-
- // sort the sample by index so that they are accessed more or less sequentially
- qsort(args->trio,args->ntrio,sizeof(trio_t),cmp_trios);
-
- free(str.s);
- free(off);
- hts_close(fp);
-}
-
-static void parse_filters(args_t *args)
-{
- if ( !args->filter_str ) return;
- int mflt = 1;
- args->nflt_str = 1;
- args->flt_str = (char**) malloc(sizeof(char*));
- args->flt_str[0] = strdup(args->filter_str);
- while (1)
- {
- int i, expanded = 0;
- for (i=args->nflt_str-1; i>=0; i--)
- {
- char *exp_beg = strchr(args->flt_str[i], '{');
- if ( !exp_beg ) continue;
- char *exp_end = strchr(exp_beg+1, '}');
- if ( !exp_end ) error("Could not parse the expression: %s\n", args->filter_str);
- char *beg = exp_beg+1, *mid = beg;
- while ( mid<exp_end )
- {
- while ( mid<exp_end && *mid!=',' ) mid++;
- kstring_t tmp = {0,0,0};
- kputsn(args->flt_str[i], exp_beg - args->flt_str[i], &tmp);
- kputsn(beg, mid - beg, &tmp);
- kputs(exp_end+1, &tmp);
- args->nflt_str++;
- hts_expand(char*, args->nflt_str, mflt, args->flt_str);
- args->flt_str[args->nflt_str-1] = tmp.s;
- beg = ++mid;
- }
- expanded = 1;
- free(args->flt_str[i]);
- memmove(&args->flt_str[i], &args->flt_str[i+1], (args->nflt_str-i-1)*sizeof(*args->flt_str));
- args->nflt_str--;
- args->flt_str[args->nflt_str] = NULL;
- }
- if ( !expanded ) break;
- }
-
- fprintf(bcftools_stderr,"Collecting data for %d filtering expressions\n", args->nflt_str);
-}
-
-static void init_data(args_t *args)
-{
- args->sr = bcf_sr_init();
- if ( args->regions )
- {
- args->sr->require_index = 1;
- if ( bcf_sr_set_regions(args->sr, args->regions, args->regions_is_file)<0 ) error("Failed to read the regions: %s\n",args->regions);
- }
- if ( args->targets && bcf_sr_set_targets(args->sr, args->targets, args->targets_is_file, 0)<0 ) error("Failed to read the targets: %s\n",args->targets);
- if ( !bcf_sr_add_reader(args->sr,args->fname) ) error("Error: %s\n", bcf_sr_strerror(args->sr->errnum));
- args->hdr = bcf_sr_get_header(args->sr,0);
-
- parse_ped(args, args->ped_fname);
- parse_filters(args);
-
- int i;
- if ( !args->nflt_str )
- {
- args->filters = (flt_stats_t*) calloc(1, sizeof(flt_stats_t));
- args->nfilters = 1;
- args->filters[0].expr = strdup("all");
- }
- else
- {
- args->nfilters = args->nflt_str;
- args->filters = (flt_stats_t*) calloc(args->nfilters, sizeof(flt_stats_t));
- for (i=0; i<args->nfilters; i++)
- {
- args->filters[i].filter = filter_init(args->hdr, args->flt_str[i]);
- args->filters[i].expr = strdup(args->flt_str[i]);
-
- // replace tab's with spaces so that the output stays parsable
- char *tmp = args->filters[i].expr;
- while ( *tmp )
- {
- if ( *tmp=='\t' ) *tmp = ' ';
- tmp++;
- }
- }
- }
- for (i=0; i<args->nfilters; i++)
- args->filters[i].stats = (trio_stats_t*) calloc(args->ntrio,sizeof(trio_stats_t));
-}
-static void destroy_data(args_t *args)
-{
- int i;
- for (i=0; i<args->nfilters; i++)
- {
- if ( args->filters[i].filter ) filter_destroy(args->filters[i].filter);
- free(args->filters[i].stats);
- free(args->filters[i].expr);
- }
- free(args->filters);
- for (i=0; i<args->nflt_str; i++) free(args->flt_str[i]);
- free(args->flt_str);
- bcf_sr_destroy(args->sr);
- free(args->trio);
- free(args->ac);
- free(args->ac_trio);
- free(args->gt_arr);
- free(args);
-}
-static void report_stats(args_t *args)
-{
- int i = 0,j;
- FILE *fh = !args->output_fname || !strcmp("-",args->output_fname) ? bcftools_stdout : fopen(args->output_fname,"w");
- if ( !fh ) error("Could not open the file for writing: %s\n", args->output_fname);
- fprintf(fh,"# CMD line shows the command line used to generate this output\n");
- fprintf(fh,"# DEF lines define expressions for all tested thresholds\n");
- fprintf(fh,"# FLT* lines report numbers for every threshold and every trio:\n");
- fprintf(fh,"# %d) filter id\n", ++i);
- fprintf(fh,"# %d) child\n", ++i);
- fprintf(fh,"# %d) father\n", ++i);
- fprintf(fh,"# %d) mother\n", ++i);
- fprintf(fh,"# %d) number of valid trio genotypes (all trio members pass filters, all non-missing)\n", ++i);
- fprintf(fh,"# %d) number of non-reference trio GTs (at least one trio member carries an alternate allele)\n", ++i);
- fprintf(fh,"# %d) number of Mendelian errors\n", ++i);
- fprintf(fh,"# %d) number of novel singleton alleles in the child (counted also as a Mendelian error)\n", ++i);
- fprintf(fh,"# %d) number of untransmitted singletons, present only in one parent\n", ++i);
- fprintf(fh,"# %d) number of transmitted singletons, present only in one parent and the child\n", ++i);
- fprintf(fh,"# %d) number of transitions, all ALT alleles present in the trio are considered\n", ++i);
- fprintf(fh,"# %d) number of transversions, all ALT alleles present in the trio are considered\n", ++i);
- fprintf(fh,"# %d) overall ts/tv, all ALT alleles present in the trio are considered\n", ++i);
- fprintf(fh, "CMD\t%s", args->argv[0]);
- for (i=1; i<args->argc; i++) fprintf(fh, " %s",args->argv[i]);
- fprintf(fh, "\n");
- for (i=0; i<args->nfilters; i++)
- {
- flt_stats_t *flt = &args->filters[i];
- fprintf(fh,"DEF\tFLT%d\t%s\n", i, flt->expr);
- }
- for (i=0; i<args->nfilters; i++)
- {
- flt_stats_t *flt = &args->filters[i];
- for (j=0; j<args->ntrio; j++)
- {
- fprintf(fh,"FLT%d", i);
- fprintf(fh,"\t%s",args->hdr->samples[args->trio[j].idx[iCHILD]]);
- fprintf(fh,"\t%s",args->hdr->samples[args->trio[j].idx[iFATHER]]);
- fprintf(fh,"\t%s",args->hdr->samples[args->trio[j].idx[iMOTHER]]);
- trio_stats_t *stats = &flt->stats[j];
- fprintf(fh,"\t%d", stats->npass);
- fprintf(fh,"\t%d", stats->nnon_ref);
- fprintf(fh,"\t%d", stats->nmendel_err);
- fprintf(fh,"\t%d", stats->nnovel);
- fprintf(fh,"\t%d", stats->nsingleton);
- fprintf(fh,"\t%d", stats->ndoubleton);
- fprintf(fh,"\t%d", stats->nts);
- fprintf(fh,"\t%d", stats->ntv);
- fprintf(fh,"\t%.2f", stats->ntv ? (float)stats->nts/stats->ntv : INFINITY);
- fprintf(fh,"\n");
- }
- }
- if ( fclose(fh)!=0 ) error("Close failed: %s\n", (!args->output_fname || !strcmp("-",args->output_fname)) ? "bcftools_stdout" : args->output_fname);
-}
-
-static inline int parse_genotype(int32_t *arr, int ngt1, int idx, int als[2])
-{
- int32_t *ptr = arr + ngt1 * idx;
- if ( bcf_gt_is_missing(ptr[0]) ) return -1;
- als[0] = bcf_gt_allele(ptr[0]);
-
- // treat haploid GTs as homozygous diploid
- if ( ngt1==1 || ptr[1]==bcf_int32_vector_end ) { als[1] = als[0]; return 0; }
-
- if ( bcf_gt_is_missing(ptr[1]) ) return -1;
- als[1] = bcf_gt_allele(ptr[1]);
-
- return 0;
-}
-
-static void process_record(args_t *args, bcf1_t *rec, flt_stats_t *flt)
-{
- int i,j;
-
- // Find out which trios pass and if the site passes
- if ( flt->filter )
- {
- uint8_t *smpl_pass = NULL;
- int pass_site = filter_test(flt->filter, rec, (const uint8_t**) &smpl_pass);
- if ( args->filter_logic & FLT_EXCLUDE )
- {
- if ( pass_site )
- {
- if ( !smpl_pass ) return;
- pass_site = 0;
- for (i=0; i<args->ntrio; i++)
- {
- int pass_trio = 1;
- for (j=0; j<3; j++)
- {
- int idx = args->trio[i].idx[j];
- if ( smpl_pass[idx] ) { pass_trio = 0; break; }
- }
- args->trio[i].pass = pass_trio;
- if ( pass_trio ) pass_site = 1;
- }
- if ( !pass_site ) return;
- }
- else
- for (i=0; i<args->ntrio; i++) args->trio[i].pass = 1;
- }
- else if ( !pass_site ) return;
- else if ( smpl_pass )
- {
- pass_site = 0;
- for (i=0; i<args->ntrio; i++)
- {
- int pass_trio = 1;
- for (j=0; j<3; j++)
- {
- int idx = args->trio[i].idx[j];
- if ( !smpl_pass[idx] ) { pass_trio = 0; break; }
- }
- args->trio[i].pass = pass_trio;
- if ( pass_trio ) pass_site = 1;
- }
- if ( !pass_site ) return;
- }
- else
- for (i=0; i<args->ntrio; i++) args->trio[i].pass = 1;
- }
-
- // Find out the allele counts. Try to use INFO/AC, if not present, determine from the genotypes
- hts_expand(int, rec->n_allele, args->mac, args->ac);
- if ( !bcf_calc_ac(args->hdr, rec, args->ac, BCF_UN_INFO|BCF_UN_FMT) ) return;
- hts_expand(int, rec->n_allele, args->mac_trio, args->ac_trio);
-
- // Get the genotypes
- int ngt = bcf_get_genotypes(args->hdr, rec, &args->gt_arr, &args->mgt_arr);
- if ( ngt<0 ) return;
- int ngt1 = ngt / rec->n_sample;
-
-
- // For ts/tv: numeric code of the reference allele, -1 for insertions
- int ref = !rec->d.allele[0][1] ? bcf_acgt2int(*rec->d.allele[0]) : -1;
-
- int star_allele = -1;
- for (i=1; i<rec->n_allele; i++)
- if ( !rec->d.allele[i][1] && rec->d.allele[i][0]=='*' ) { star_allele = i; break; }
-
- // Run the stats
- for (i=0; i<args->ntrio; i++)
- {
- if ( flt->filter && !args->trio[i].pass ) continue;
- trio_stats_t *stats = &flt->stats[i];
-
- // Determine the alternate allele and the genotypes, skip if any of the alleles is missing.
- // the order is: child, father, mother
- int als[6], *als_child = als, *als_father = als+2, *als_mother = als+4;
- if ( parse_genotype(args->gt_arr, ngt1, args->trio[i].idx[iCHILD], als_child) < 0 ) continue;
- if ( parse_genotype(args->gt_arr, ngt1, args->trio[i].idx[iFATHER], als_father) < 0 ) continue;
- if ( parse_genotype(args->gt_arr, ngt1, args->trio[i].idx[iMOTHER], als_mother) < 0 ) continue;
-
- stats->npass++;
-
- // Has the trio an alternate allele other than *?
- int has_star_allele = 0, has_nonref = 0;
- memset(args->ac_trio,0,rec->n_allele*sizeof(*args->ac_trio));
- for (j=0; j<6; j++)
- {
- if ( als[j]==star_allele ) { has_star_allele = 1; continue; }
- if ( als[j]==0 ) continue;
- has_nonref = 1;
- args->ac_trio[ als[j] ]++;
- }
- if ( !has_nonref ) continue; // only ref or * in this trio
-
- stats->nnon_ref++;
-
- // Calculate ts/tv. It does the right thing and handles also HetAA genotypes
- if ( ref != -1 )
- {
- int has_ts = 0, has_tv = 0;
- for (j=0; j<6; j++)
- {
- if ( als[j]==0 || als[j]==star_allele ) continue;
- if ( als[j] >= rec->n_allele )
- error("The GT index is out of range at %s:%d in %s\n", bcf_seqname(args->hdr,rec),rec->pos+1,args->hdr->samples[args->trio[i].idx[j/2]]);
- if ( rec->d.allele[als[j]][1] ) continue;
-
- int alt = bcf_acgt2int(rec->d.allele[als[j]][0]);
- if ( abs(ref-alt)==2 ) has_ts = 1;
- else has_tv = 1;
- }
- if ( has_ts ) stats->nts++;
- if ( has_tv ) stats->ntv++;
- }
-
- // Skip some stats if the star allele is present as it was already checked at the primary record, we do not want to count the same
- // thing multiple times. There can be other alternate allele, but we ignore that for simplicity.
- if ( has_star_allele ) continue;
-
- // Detect mendelian errors
- int mendel_ok = (als_child[0]==als_father[0] || als_child[0]==als_father[1]) && (als_child[1]==als_mother[0] || als_child[1]==als_mother[1]) ? 1 : 0;
- if ( !mendel_ok ) mendel_ok = (als_child[1]==als_father[0] || als_child[1]==als_father[1]) && (als_child[0]==als_mother[0] || als_child[0]==als_mother[1]) ? 1 : 0;
- if ( !mendel_ok ) stats->nmendel_err++;
-
- // Is this a singleton, doubleton, neither?
- for (j=1; j<rec->n_allele; j++)
- {
- if ( args->ac_trio[j]==1 && args->ac[j]==1 ) // singleton (in parent) or novel (in child)
- {
- if ( als_child[0]==j || als_child[1]==j ) stats->nnovel++;
- else stats->nsingleton++;
- }
- else if ( args->ac_trio[j]==2 && args->ac[j]==2 ) // possibly a doubleton
- {
- if ( (als_child[0]==j || als_child[1]==j) && (als_child[0]!=j || als_child[1]!=j) ) stats->ndoubleton++;
- }
- }
- }
-}
-
-int run(int argc, char **argv)
-{
- args_t *args = (args_t*) calloc(1,sizeof(args_t));
- args->argc = argc; args->argv = argv;
- args->output_fname = "-";
- static struct option loptions[] =
- {
- {"include",required_argument,0,'i'},
- {"exclude",required_argument,0,'e'},
- {"output",required_argument,NULL,'o'},
- {"ped",required_argument,NULL,'p'},
- {"regions",1,0,'r'},
- {"regions-file",1,0,'R'},
- {"targets",1,0,'t'},
- {"targets-file",1,0,'T'},
- {NULL,0,NULL,0}
- };
- int c, i;
- while ((c = getopt_long(argc, argv, "p:o:s:i:e:r:R:t:T:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'e': args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
- case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
- case 't': args->targets = optarg; break;
- case 'T': args->targets = optarg; args->targets_is_file = 1; break;
- case 'r': args->regions = optarg; break;
- case 'R': args->regions = optarg; args->regions_is_file = 1; break;
- case 'o': args->output_fname = optarg; break;
- case 'p': args->ped_fname = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage_text()); break;
- }
- }
- if ( optind==argc )
- {
- if ( !isatty(fileno((FILE *)stdin)) ) args->fname = "-"; // reading from stdin
- else { error("%s", usage_text()); }
- }
- else if ( optind+1!=argc ) error("%s", usage_text());
- else args->fname = argv[optind];
-
- if ( !args->ped_fname ) error("Missing the -p, --ped option\n");
-
- init_data(args);
-
- while ( bcf_sr_next_line(args->sr) )
- {
- bcf1_t *rec = bcf_sr_get_line(args->sr,0);
- for (i=0; i<args->nfilters; i++)
- process_record(args, rec, &args->filters[i]);
- }
-
- report_stats(args);
- destroy_data(args);
-
- return 0;
-}
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/khash_str2int.h>
-#include "bcftools.h"
-
-typedef struct
-{
- int father, mother, child; // VCF sample index
- int prev, ipop;
- uint32_t err, nswitch, ntest;
-}
-trio_t;
-
-typedef struct
-{
- char *name;
- uint32_t err, nswitch, ntest, ntrio;
- float pswitch;
-}
-pop_t;
-
-typedef struct
-{
- int argc;
- char **argv;
- bcf_hdr_t *hdr;
- trio_t *trio;
- int ntrio, mtrio;
- int32_t *gt_arr;
- int npop;
- pop_t *pop;
- int mgt_arr, prev_rid;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Calculate phase switch rate in trio samples, children samples must have phased GTs.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Calculate phase switch rate in trio children.\n"
- "Usage: bcftools +trio-swich-rate [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -p, --ped <file> PED file with optional 7th column to group\n"
- " results by population\n"
- "\n"
- "Example:\n"
- " bcftools +trio-switch-rate file.bcf -- -p file.ped\n"
- "\n";
-}
-
-void parse_ped(args_t *args, char *fname)
-{
- htsFile *fp = hts_open(fname, "r");
- if ( !fp ) error("Could not read: %s\n", fname);
-
- kstring_t str = {0,0,0};
- if ( hts_getline(fp, KS_SEP_LINE, &str) <= 0 ) error("Empty file: %s\n", fname);
-
- void *pop2i = khash_str2int_init();
-
- int moff = 0, *off = NULL;
- do
- {
- // familyID sampleID paternalID maternalID sex phenotype population relationship siblings secondOrder thirdOrder children comment
- // BB03 HG01884 HG01885 HG01956 2 0 ACB child 0 0 0 0
- int ncols = ksplit_core(str.s,0,&moff,&off);
- if ( ncols<4 ) error("Could not parse the ped file: %s\n", str.s);
-
- int father = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[2]]);
- if ( father<0 ) continue;
- int mother = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[3]]);
- if ( mother<0 ) continue;
- int child = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[1]]);
- if ( child<0 ) continue;
-
- args->ntrio++;
- hts_expand0(trio_t,args->ntrio,args->mtrio,args->trio);
- trio_t *trio = &args->trio[args->ntrio-1];
- trio->father = father;
- trio->mother = mother;
- trio->child = child;
-
- if (ncols>6) {
- char *pop_name = &str.s[off[6]];
- if ( !khash_str2int_has_key(pop2i,pop_name) )
- {
- pop_name = strdup(&str.s[off[6]]);
- khash_str2int_set(pop2i,pop_name,args->npop);
- args->npop++;
- args->pop = (pop_t*) realloc(args->pop,args->npop*sizeof(*args->pop));
- memset(args->pop+args->npop-1,0,sizeof(*args->pop));
- args->pop[args->npop-1].name = pop_name;
- }
- khash_str2int_get(pop2i,pop_name,&trio->ipop);
- args->pop[trio->ipop].ntrio++;
- }
- } while ( hts_getline(fp, KS_SEP_LINE, &str)>=0 );
-
- khash_str2int_destroy(pop2i);
- free(str.s);
- free(off);
- hts_close(fp);
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- memset(&args,0,sizeof(args_t));
- args.argc = argc; args.argv = argv;
- args.prev_rid = -1;
- args.hdr = in;
- char *ped_fname = NULL;
- static struct option loptions[] =
- {
- {"ped",required_argument,NULL,'p'},
- {0,0,0,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?hp:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'p': ped_fname = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( !ped_fname ) error("Expected the -p option\n");
- parse_ped(&args, ped_fname);
- return 1;
-}
-
-typedef struct
-{
- int a, b, phased;
-}
-gt_t;
-
-int parse_genotype(gt_t *gt, int32_t *ptr);
-
-inline int parse_genotype(gt_t *gt, int32_t *ptr)
-{
- if ( ptr[0]==bcf_gt_missing ) return 0;
- if ( ptr[1]==bcf_gt_missing ) return 0;
- if ( ptr[1]==bcf_int32_vector_end ) return 0;
- gt->phased = bcf_gt_is_phased(ptr[1]) ? 1 : 0;
- gt->a = bcf_gt_allele(ptr[0]); if ( gt->a > 1 ) return 0; // consider only the first two alleles at biallelic sites
- gt->b = bcf_gt_allele(ptr[1]); if ( gt->b > 1 ) return 0;
- return 1;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int ngt = bcf_get_genotypes(args.hdr, rec, &args.gt_arr, &args.mgt_arr);
- if ( ngt<0 ) return NULL;
- ngt /= bcf_hdr_nsamples(args.hdr);
- if ( ngt!=2 ) return NULL;
-
- int i;
- if ( rec->rid!=args.prev_rid )
- {
- args.prev_rid = rec->rid;
- for (i=0; i<args.ntrio; i++) args.trio[i].prev = 0;
- }
-
- gt_t child, father, mother;
- for (i=0; i<args.ntrio; i++)
- {
- trio_t *trio = &args.trio[i];
-
- if ( !parse_genotype(&child, args.gt_arr + ngt*trio->child) ) continue;
- if ( !child.phased ) continue;
- if ( child.a+child.b != 1 ) continue; // child is not a het
-
- if ( !parse_genotype(&father, args.gt_arr + ngt*trio->father) ) continue;
- if ( !parse_genotype(&mother, args.gt_arr + ngt*trio->mother) ) continue;
- if ( father.a+father.b == 1 && mother.a+mother.b == 1 ) continue; // both parents are hets
- if ( father.a+father.b == mother.a+mother.b ) { trio->err++; continue; } // mendelian error
-
- int test_phase = 0;
- if ( father.a==father.b ) test_phase = 1 + (child.a==father.a);
- else if ( mother.a==mother.b ) test_phase = 1 + (child.b==mother.a);
- if ( trio->prev > 0 )
- {
- if ( trio->prev!=test_phase ) trio->nswitch++;
- }
- trio->ntest++;
- trio->prev = test_phase;
- }
- return NULL;
-}
-
-void destroy(void)
-{
- int i;
- printf("# This file was produced by: bcftools +trio-switch-rate(%s+htslib-%s)\n", bcftools_version(),hts_version());
- printf("# The command line was:\tbcftools +trio-switch-rate %s", args.argv[0]);
- for (i=1; i<args.argc; i++) printf(" %s",args.argv[i]);
- printf("\n#\n");
- printf("# TRIO\t[2]Father\t[3]Mother\t[4]Child\t[5]nTested\t[6]nMendelian Errors\t[7]nSwitch\t[8]nSwitch (%%)\n");
- for (i=0; i<args.ntrio; i++)
- {
- trio_t *trio = &args.trio[i];
- printf("TRIO\t%s\t%s\t%s\t%d\t%d\t%d\t%.2f\n",
- bcf_hdr_int2id(args.hdr,BCF_DT_SAMPLE,trio->father),
- bcf_hdr_int2id(args.hdr,BCF_DT_SAMPLE,trio->mother),
- bcf_hdr_int2id(args.hdr,BCF_DT_SAMPLE,trio->child),
- trio->ntest, trio->err, trio->nswitch, trio->ntest ? trio->nswitch*100./trio->ntest : 0
- );
- if (args.npop) {
- pop_t *pop = &args.pop[trio->ipop];
- pop->err += trio->err;
- pop->nswitch += trio->nswitch;
- pop->ntest += trio->ntest;
- pop->pswitch += trio->ntest ? trio->nswitch*100./trio->ntest : 0;
- }
- }
- printf("# POP\tpopulation or other grouping defined by an optional 7-th column of the PED file\n");
- printf("# POP\t[2]Name\t[3]Number of trios\t[4]avgTested\t[5]avgMendelian Errors\t[6]avgSwitch\t[7]avgSwitch (%%)\n");
- for (i=0; i<args.npop; i++)
- {
- pop_t *pop = &args.pop[i];
- printf("POP\t%s\t%d\t%.0f\t%.0f\t%.0f\t%.2f\n", pop->name,pop->ntrio,
- (float)pop->ntest/pop->ntrio,(float)pop->err/pop->ntrio,(float)pop->nswitch/pop->ntrio,
- pop->pswitch/pop->ntrio);
- }
- for (i=0; i<args.npop; i++) free(args.pop[i].name);
- free(args.pop);
- free(args.trio);
- free(args.gt_arr);
-}
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2016 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <getopt.h>
-#include <math.h>
-#include <htslib/hts.h>
-#include <htslib/vcf.h>
-#include <htslib/kstring.h>
-#include <htslib/kseq.h>
-#include <htslib/khash_str2int.h>
-#include "bcftools.h"
-
-typedef struct
-{
- int father, mother, child; // VCF sample index
- int prev, ipop;
- uint32_t err, nswitch, ntest;
-}
-trio_t;
-
-typedef struct
-{
- char *name;
- uint32_t err, nswitch, ntest, ntrio;
- float pswitch;
-}
-pop_t;
-
-typedef struct
-{
- int argc;
- char **argv;
- bcf_hdr_t *hdr;
- trio_t *trio;
- int ntrio, mtrio;
- int32_t *gt_arr;
- int npop;
- pop_t *pop;
- int mgt_arr, prev_rid;
-}
-args_t;
-
-args_t args;
-
-const char *about(void)
-{
- return "Calculate phase switch rate in trio samples, children samples must have phased GTs.\n";
-}
-
-const char *usage(void)
-{
- return
- "\n"
- "About: Calculate phase switch rate in trio children.\n"
- "Usage: bcftools +trio-swich-rate [General Options] -- [Plugin Options]\n"
- "Options:\n"
- " run \"bcftools plugin\" for a list of common options\n"
- "\n"
- "Plugin options:\n"
- " -p, --ped <file> PED file with optional 7th column to group\n"
- " results by population\n"
- "\n"
- "Example:\n"
- " bcftools +trio-switch-rate file.bcf -- -p file.ped\n"
- "\n";
-}
-
-void parse_ped(args_t *args, char *fname)
-{
- htsFile *fp = hts_open(fname, "r");
- if ( !fp ) error("Could not read: %s\n", fname);
-
- kstring_t str = {0,0,0};
- if ( hts_getline(fp, KS_SEP_LINE, &str) <= 0 ) error("Empty file: %s\n", fname);
-
- void *pop2i = khash_str2int_init();
-
- int moff = 0, *off = NULL;
- do
- {
- // familyID sampleID paternalID maternalID sex phenotype population relationship siblings secondOrder thirdOrder children comment
- // BB03 HG01884 HG01885 HG01956 2 0 ACB child 0 0 0 0
- int ncols = ksplit_core(str.s,0,&moff,&off);
- if ( ncols<4 ) error("Could not parse the ped file: %s\n", str.s);
-
- int father = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[2]]);
- if ( father<0 ) continue;
- int mother = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[3]]);
- if ( mother<0 ) continue;
- int child = bcf_hdr_id2int(args->hdr,BCF_DT_SAMPLE,&str.s[off[1]]);
- if ( child<0 ) continue;
-
- args->ntrio++;
- hts_expand0(trio_t,args->ntrio,args->mtrio,args->trio);
- trio_t *trio = &args->trio[args->ntrio-1];
- trio->father = father;
- trio->mother = mother;
- trio->child = child;
-
- if (ncols>6) {
- char *pop_name = &str.s[off[6]];
- if ( !khash_str2int_has_key(pop2i,pop_name) )
- {
- pop_name = strdup(&str.s[off[6]]);
- khash_str2int_set(pop2i,pop_name,args->npop);
- args->npop++;
- args->pop = (pop_t*) realloc(args->pop,args->npop*sizeof(*args->pop));
- memset(args->pop+args->npop-1,0,sizeof(*args->pop));
- args->pop[args->npop-1].name = pop_name;
- }
- khash_str2int_get(pop2i,pop_name,&trio->ipop);
- args->pop[trio->ipop].ntrio++;
- }
- } while ( hts_getline(fp, KS_SEP_LINE, &str)>=0 );
-
- khash_str2int_destroy(pop2i);
- free(str.s);
- free(off);
- hts_close(fp);
-}
-
-int init(int argc, char **argv, bcf_hdr_t *in, bcf_hdr_t *out)
-{
- memset(&args,0,sizeof(args_t));
- args.argc = argc; args.argv = argv;
- args.prev_rid = -1;
- args.hdr = in;
- char *ped_fname = NULL;
- static struct option loptions[] =
- {
- {"ped",required_argument,NULL,'p'},
- {0,0,0,0}
- };
- int c;
- while ((c = getopt_long(argc, argv, "?hp:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 'p': ped_fname = optarg; break;
- case 'h':
- case '?':
- default: error("%s", usage()); break;
- }
- }
- if ( !ped_fname ) error("Expected the -p option\n");
- parse_ped(&args, ped_fname);
- return 1;
-}
-
-typedef struct
-{
- int a, b, phased;
-}
-gt_t;
-
-int parse_genotype(gt_t *gt, int32_t *ptr);
-
-inline int parse_genotype(gt_t *gt, int32_t *ptr)
-{
- if ( ptr[0]==bcf_gt_missing ) return 0;
- if ( ptr[1]==bcf_gt_missing ) return 0;
- if ( ptr[1]==bcf_int32_vector_end ) return 0;
- gt->phased = bcf_gt_is_phased(ptr[1]) ? 1 : 0;
- gt->a = bcf_gt_allele(ptr[0]); if ( gt->a > 1 ) return 0; // consider only the first two alleles at biallelic sites
- gt->b = bcf_gt_allele(ptr[1]); if ( gt->b > 1 ) return 0;
- return 1;
-}
-
-bcf1_t *process(bcf1_t *rec)
-{
- int ngt = bcf_get_genotypes(args.hdr, rec, &args.gt_arr, &args.mgt_arr);
- if ( ngt<0 ) return NULL;
- ngt /= bcf_hdr_nsamples(args.hdr);
- if ( ngt!=2 ) return NULL;
-
- int i;
- if ( rec->rid!=args.prev_rid )
- {
- args.prev_rid = rec->rid;
- for (i=0; i<args.ntrio; i++) args.trio[i].prev = 0;
- }
-
- gt_t child, father, mother;
- for (i=0; i<args.ntrio; i++)
- {
- trio_t *trio = &args.trio[i];
-
- if ( !parse_genotype(&child, args.gt_arr + ngt*trio->child) ) continue;
- if ( !child.phased ) continue;
- if ( child.a+child.b != 1 ) continue; // child is not a het
-
- if ( !parse_genotype(&father, args.gt_arr + ngt*trio->father) ) continue;
- if ( !parse_genotype(&mother, args.gt_arr + ngt*trio->mother) ) continue;
- if ( father.a+father.b == 1 && mother.a+mother.b == 1 ) continue; // both parents are hets
- if ( father.a+father.b == mother.a+mother.b ) { trio->err++; continue; } // mendelian error
-
- int test_phase = 0;
- if ( father.a==father.b ) test_phase = 1 + (child.a==father.a);
- else if ( mother.a==mother.b ) test_phase = 1 + (child.b==mother.a);
- if ( trio->prev > 0 )
- {
- if ( trio->prev!=test_phase ) trio->nswitch++;
- }
- trio->ntest++;
- trio->prev = test_phase;
- }
- return NULL;
-}
-
-void destroy(void)
-{
- int i;
- fprintf(bcftools_stdout, "# This file was produced by: bcftools +trio-switch-rate(%s+htslib-%s)\n", bcftools_version(),hts_version());
- fprintf(bcftools_stdout, "# The command line was:\tbcftools +trio-switch-rate %s", args.argv[0]);
- for (i=1; i<args.argc; i++) fprintf(bcftools_stdout, " %s",args.argv[i]);
- fprintf(bcftools_stdout, "\n#\n");
- fprintf(bcftools_stdout, "# TRIO\t[2]Father\t[3]Mother\t[4]Child\t[5]nTested\t[6]nMendelian Errors\t[7]nSwitch\t[8]nSwitch (%%)\n");
- for (i=0; i<args.ntrio; i++)
- {
- trio_t *trio = &args.trio[i];
- fprintf(bcftools_stdout, "TRIO\t%s\t%s\t%s\t%d\t%d\t%d\t%.2f\n",
- bcf_hdr_int2id(args.hdr,BCF_DT_SAMPLE,trio->father),
- bcf_hdr_int2id(args.hdr,BCF_DT_SAMPLE,trio->mother),
- bcf_hdr_int2id(args.hdr,BCF_DT_SAMPLE,trio->child),
- trio->ntest, trio->err, trio->nswitch, trio->ntest ? trio->nswitch*100./trio->ntest : 0
- );
- if (args.npop) {
- pop_t *pop = &args.pop[trio->ipop];
- pop->err += trio->err;
- pop->nswitch += trio->nswitch;
- pop->ntest += trio->ntest;
- pop->pswitch += trio->ntest ? trio->nswitch*100./trio->ntest : 0;
- }
- }
- fprintf(bcftools_stdout, "# POP\tpopulation or other grouping defined by an optional 7-th column of the PED file\n");
- fprintf(bcftools_stdout, "# POP\t[2]Name\t[3]Number of trios\t[4]avgTested\t[5]avgMendelian Errors\t[6]avgSwitch\t[7]avgSwitch (%%)\n");
- for (i=0; i<args.npop; i++)
- {
- pop_t *pop = &args.pop[i];
- fprintf(bcftools_stdout, "POP\t%s\t%d\t%.0f\t%.0f\t%.0f\t%.2f\n", pop->name,pop->ntrio,
- (float)pop->ntest/pop->ntrio,(float)pop->err/pop->ntrio,(float)pop->nswitch/pop->ntrio,
- pop->pswitch/pop->ntrio);
- }
- for (i=0; i<args.npop; i++) free(args.pop[i].name);
- free(args.pop);
- free(args.trio);
- free(args.gt_arr);
-}
}
free(str.s);
- hts_close(fp);
+ if ( hts_close(fp)!=0 )
+ {
+ fprintf(stderr,"[%s] Error: close failed .. %s\n", __func__,fname);
+ goto error;
+ }
return idx;
error:
{
int iend = iBIN(end);
if ( iend > list->nidx ) iend = list->nidx;
- for (i=ibeg; i<iend; i++)
+ for (i=ibeg; i<=iend; i++)
if ( list->idx[i] ) break;
- if ( i==iend ) return 0;
+ if ( i>iend ) return 0;
i = list->idx[i];
}
-
for (ireg=i-1; ireg<list->nreg; ireg++)
{
if ( list->reg[ireg].beg > end ) return 0; // no match, past the query region
}
free(str.s);
- hts_close(fp);
+ if ( hts_close(fp)!=0 )
+ {
+ fprintf(bcftools_stderr,"[%s] Error: close failed .. %s\n", __func__,fname);
+ goto error;
+ }
return idx;
error:
{
int iend = iBIN(end);
if ( iend > list->nidx ) iend = list->nidx;
- for (i=ibeg; i<iend; i++)
+ for (i=ibeg; i<=iend; i++)
if ( list->idx[i] ) break;
- if ( i==iend ) return 0;
+ if ( i>iend ) return 0;
i = list->idx[i];
}
-
for (ireg=i-1; ireg<list->nreg; ireg++)
{
if ( list->reg[ireg].beg > end ) return 0; // no match, past the query region
// and for working example see test/test-regidx.c.
regidx_t *idx = regidx_init(in_fname,parse_custom,free_custom,sizeof(char*),NULL);
- // Query overlap with chr:from-to
+ // Query overlap with chr:beg-end (beg,end are 1-based coordinates)
regitr_t *itr = regitr_init(idx);
- if ( regidx_overlap(idx, chr,from,to, itr) ) printf("There is an overlap!\n");
+ if ( regidx_overlap(idx, chr,beg-1,end-1, itr) ) printf("There is an overlap!\n");
while ( regitr_overlap(itr) )
{
- printf("[%d,%d] overlaps with [%d,%d], payload=%s\n", from,to,
- itr->beg, itr->end, regitr_payload(itr,char*));
+ printf("[%d,%d] overlaps with [%d,%d], payload=%s\n", beg,end,
+ itr->beg+1, itr->end+1, regitr_payload(itr,char*));
}
regidx_destroy(idx);
regitr_t *itr = regitr_init(idx);
while ( regitr_loop(itr) )
- printf("chr=%s beg=%d end=%d\n", itr->seq, itr->beg, itr->end);
+ printf("chr=%s beg=%d end=%d\n", itr->seq, itr->beg+1, itr->end+1);
regidx_destroy(idx);
regitr_destroy(itr);
#include <inttypes.h>
#include <fcntl.h>
#include <math.h>
+#ifdef _WIN32
+#include <windows.h>
+#endif
#include <htslib/vcf.h>
#include <htslib/bgzf.h>
#include <htslib/tbx.h> // for hts_get_bgzfp()
#include <htslib/kseq.h>
#include <htslib/thread_pool.h>
+#include <htslib/faidx.h>
+#include <htslib/khash_str2int.h>
#include "bcftools.h"
#include "khash_str2str.h"
typedef struct _args_t
{
char **argv, *fname, *samples_fname, *header_fname, *output_fname;
+ char *fai_fname, *rm_tmpfile;
htsFile *fp;
htsFormat type;
htsThreadPool *threads;
}
args_t;
+static inline int is_escaped(const char *min, const char *str)
+{
+ int n = 0;
+ while ( --str>=min && *str=='\\' ) n++;
+ return n%2;
+}
+static char *copy_and_update_contig_line(faidx_t *fai, char *line, void *chr_seen, kstring_t *dst)
+{
+ kstring_t key = {0,0,0}, val = {0,0,0}, tmp = {0,0,0};
+ char *chr_name = NULL, *p, *q = line + 9; // skip ##contig=
+ char *end = q;
+ int nopen = 1, chr_len = 0;
+ while ( *end && *end!='\n' ) end++;
+ while ( *q && *q!='\n' && nopen>0 )
+ {
+ p = ++q;
+ while ( *q && (*q==' ' || *q=='\t') ) { p++; q++; }
+ // ^[A-Za-z_][0-9A-Za-z_.]*$
+ if (p==q && *q && (isalpha(*q) || *q=='_'))
+ {
+ q++;
+ while ( *q && (isalnum(*q) || *q=='_' || *q=='.') ) q++;
+ }
+ int n = q-p;
+ int m = 0;
+ while ( *q && (*q==' ' || *q=='\t') ) { q++; m++; }
+ if ( *q!='=' || !n )
+ {
+ char *x = q;
+ while ( *x && *x!='\n' ) x++;
+ *x = '\0';
+ error("Could not parse the line: %s [%s][%s]\n", line,p,q);
+ }
+ key.l = 0;
+ kputsn(p,q-p-m,&key);
+ p = ++q;
+ while ( *q && (*q==' ' || *q=='\t') ) { p++; q++; }
+ int quoted = *p=='"' ? 1 : 0;
+ if ( quoted ) p++, q++;
+ while ( *q && *q != '\n' )
+ {
+ if ( quoted ) { if ( *q=='"' && !is_escaped(p,q) ) break; }
+ else
+ {
+ if ( *q=='<' ) nopen++;
+ if ( *q=='>' ) nopen--;
+ if ( !nopen ) break;
+ if ( *q==',' && nopen==1 ) break;
+ }
+ q++;
+ }
+ char *r = q;
+ while ( r > p && r[-1] == ' ' ) r--;
+ val.l = 0;
+ kputsn(p,r-p,&val);
+ if ( quoted && *q=='"' ) q++;
+ if ( *q=='>' ) { nopen--; q++; }
+ if ( !strcmp("length",key.s) ) continue;
+ if ( !strcmp("ID",key.s) )
+ {
+ if ( khash_str2int_has_key(chr_seen,val.s) ) continue;
+ chr_len = faidx_seq_len(fai, val.s);
+ if ( chr_len==-1 )
+ {
+ free(val.s); free(key.s); free(tmp.s);
+ return end; // the sequence is not in fai, remove
+ }
+ chr_name = strdup(val.s);
+ khash_str2int_inc(chr_seen, chr_name);
+ continue;
+ }
+ kputc(',',&tmp);
+ kputs(key.s,&tmp);
+ kputc('=',&tmp);
+ if ( quoted ) kputc('"',&tmp);
+ kputs(val.s,&tmp);
+ if ( quoted ) kputc('"',&tmp);
+ }
+ if ( !chr_name ) return end;
+ ksprintf(dst,"##contig=<ID=%s,length=%d%s>",chr_name,chr_len,tmp.l ? tmp.s : "");
+ free(key.s); free(val.s); free(tmp.s);
+ return q;
+}
+static void update_from_fai(args_t *args)
+{
+ if ( !strcmp("-",args->fname) )
+ error("Cannot use the --fai option when reading from standard input.\n");
+
+ faidx_t *fai = fai_load3(args->fai_fname,args->fai_fname,NULL,FAI_FASTA);
+ if ( !fai ) error("Could not parse %s\n", args->fai_fname);
+#ifdef _WIN32
+ char tmp_path[MAX_PATH];
+ int ret = GetTempPath(MAX_PATH, tmp_path);
+ if (!ret || ret > MAX_PATH)
+ error("Could not get the path to the temporary folder\n");
+ if (strlen(tmp_path) + strlen("/bcftools-fai-header-XXXXXX") >= MAX_PATH)
+ error("Full path to the temporary folder is too long\n");
+ strcat(tmp_path, "/bcftools-fai-header-XXXXXX");
+ args->rm_tmpfile = strdup(tmp_path);
+#else
+ args->rm_tmpfile = strdup("/tmp/bcftools-fai-header-XXXXXX");
+#endif
+ int fd = mkstemp(args->rm_tmpfile);
+ if ( fd<0 ) error("Could not open a temporary file for writing: %s\n", args->rm_tmpfile);
+
+ // get a template header: either from the original VCF or from --header
+ char *ori_hdr_fname = args->header_fname ? args->header_fname : args->fname;
+ htsFile *fp = hts_open(ori_hdr_fname,"r");
+ if ( !fp ) error("Failed to open: %s\n", ori_hdr_fname);
+ bcf_hdr_t *hdr = bcf_hdr_read(fp);
+ if ( !hdr ) error("Failed to read the header: %s\n", ori_hdr_fname);
+ hts_close(fp); // no need to check the return status here
+
+ // put the header in a text buffer
+ kstring_t hdr_txt_ori = {0,0,0}, hdr_txt_new = {0,0,0};
+ bcf_hdr_format(hdr, 0, &hdr_txt_ori);
+ bcf_hdr_destroy(hdr);
+
+ // update the existing contig lines and remove lines not present in the fai file
+ void *chr_seen = khash_str2int_init();
+ char *tmp, *beg = hdr_txt_ori.s;
+ while ( beg && *beg )
+ {
+ tmp = strstr(beg, "\n##contig=<");
+ if ( !tmp ) break;
+ kputsn(beg, tmp-beg+1, &hdr_txt_new);
+ size_t l_prev = hdr_txt_new.l;
+ beg = copy_and_update_contig_line(fai,tmp+1,chr_seen, &hdr_txt_new);
+ if ( l_prev==hdr_txt_new.l ) hdr_txt_new.l--; // nothing was added, remove the newline
+ }
+ if ( !beg || !(tmp=strstr(beg,"\n#CHROM")) ) error("Failed to parse the header, #CHROM not found\n");
+ kputsn(beg, tmp-beg+1, &hdr_txt_new);
+
+ // add any new contig lines
+ int i, n = faidx_nseq(fai);
+ for (i=0; i<n; i++)
+ {
+ if ( khash_str2int_has_key(chr_seen,faidx_iseq(fai,i)) ) continue;
+ ksprintf(&hdr_txt_new,"##contig=<ID=%s,length=%d>\n",faidx_iseq(fai,i),faidx_seq_len(fai,faidx_iseq(fai,i)));
+ }
+ kputs(tmp+1,&hdr_txt_new);
+
+ if ( write(fd, hdr_txt_new.s, hdr_txt_new.l)!=hdr_txt_new.l ) error("Failed to write %zu bytes to %s\n", hdr_txt_new.l,args->rm_tmpfile);
+ if ( close(fd)!=0 ) error("Failed to close %s\n", args->rm_tmpfile);
+ args->header_fname = args->rm_tmpfile;
+
+ free(hdr_txt_ori.s);
+ free(hdr_txt_new.s);
+ fai_destroy(fai);
+ khash_str2int_destroy_free(chr_seen);
+}
+
static void read_header_file(char *fname, kstring_t *hdr)
{
kstring_t tmp = {0,0,0};
kputc('\n',&fp->line);
if ( write(out, fp->line.s, fp->line.l)!=fp->line.l ) error("Failed to write %"PRIu64" bytes\n", (uint64_t)fp->line.l);
}
- hts_close(fp);
- close(out);
+ if ( hts_close(fp)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fname);
+ if ( close(out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
}
static bcf_hdr_t *strip_header(bcf_hdr_t *src, bcf_hdr_t *dst)
if ( j>=0 )
{
j = atoi(src_hrec->vals[j]);
- hrec_add_idx(tmp, j);
+ if (hrec_add_idx(tmp, j) < 0)
+ error_errno("[%s] Failed to add IDX header", __func__);
}
bcf_hdr_add_hrec(out, tmp);
}
}
- bcf_hdr_sync(out);
+ if (bcf_hdr_sync(out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
for (i=0; i<dst->nhrec; i++)
{
// finally add new structured fields
if ( args->n_threads > 0 )
{
- args->threads = calloc(1, sizeof(*args->threads));
+ args->threads = (htsThreadPool *) calloc(1, sizeof(htsThreadPool));
if ( !args->threads ) error("Could not allocate memory\n");
if ( !(args->threads->pool = hts_tpool_init(args->n_threads)) ) error("Could not initialize threading\n");
- BGZF *bgzf = hts_get_bgzfp(fp);
- if ( bgzf ) bgzf_thread_pool(bgzf, args->threads->pool, args->threads->qsize);
+ hts_set_thread_pool(fp, args->threads);
}
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to read the header: %s\n", args->fname);
htsFile *fp_out = hts_open(args->output_fname ? args->output_fname : "-",is_compressed ? "wb" : "wbu");
if ( !fp_out ) error("%s: %s\n", args->output_fname ? args->output_fname : "-", strerror(errno));
if ( args->threads )
- {
- BGZF *bgzf = hts_get_bgzfp(fp_out);
- if ( bgzf ) bgzf_thread_pool(bgzf, args->threads->pool, args->threads->qsize);
- }
- bcf_hdr_write(fp_out, hdr_out);
+ hts_set_thread_pool(fp_out, args->threads);
+ if ( bcf_hdr_write(fp_out, hdr_out)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname ? args->output_fname : "standard output");
bcf1_t *rec = bcf_init();
while ( bcf_read(fp, hdr, rec)==0 )
if ( i!=rec->n_fmt )
error("The FORMAT tag is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.fmt[i].id));
- bcf_write(fp_out,hdr_out,rec);
+ if ( bcf_write(fp_out,hdr_out,rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname ? args->output_fname : "standard output");
}
bcf_destroy(rec);
free(htxt.s);
- hts_close(fp_out);
- hts_close(fp);
+ if ( hts_close(fp_out)!=0 ) error("[%s] Error: failed to close the file %s\n",__func__,args->output_fname ? args->output_fname : "standard output");
+ if ( hts_close(fp)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fname);
bcf_hdr_destroy(hdr_out);
bcf_hdr_destroy(hdr);
if ( args->threads )
fprintf(stderr, "Usage: bcftools reheader [OPTIONS] <in.vcf.gz>\n");
fprintf(stderr, "\n");
fprintf(stderr, "Options:\n");
+ fprintf(stderr, " -f, --fai <file> update sequences and their lengths from the .fai file\n");
fprintf(stderr, " -h, --header <file> new header\n");
fprintf(stderr, " -o, --output <file> write output to a file [standard output]\n");
fprintf(stderr, " -s, --samples <file> new sample names\n");
- fprintf(stderr, " --threads <int> number of extra compression threads (BCF only) [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads (BCF only) [0]\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Example:\n");
+ fprintf(stderr, " # Write out the header to be modified\n");
+ fprintf(stderr, " bcftools view -h old.bcf > header.txt\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " # Edit the header using your favorite text editor\n");
+ fprintf(stderr, " vi header.txt\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " # Reheader the file\n");
+ fprintf(stderr, " bcftools reheader -h header.txt -o new.bcf old.bcf\n");
fprintf(stderr, "\n");
exit(1);
}
static struct option loptions[] =
{
+ {"fai",1,0,'f'},
{"output",1,0,'o'},
{"header",1,0,'h'},
{"samples",1,0,'s'},
{"threads",1,NULL,1},
{0,0,0,0}
};
- while ((c = getopt_long(argc, argv, "s:h:o:",loptions,NULL)) >= 0)
+ while ((c = getopt_long(argc, argv, "s:h:o:f:",loptions,NULL)) >= 0)
{
switch (c)
{
case 1 : args->n_threads = strtol(optarg, 0, 0); break;
+ case 'f': args->fai_fname = optarg; break;
case 'o': args->output_fname = optarg; break;
case 's': args->samples_fname = optarg; break;
case 'h': args->header_fname = optarg; break;
- case '?': usage(args);
+ case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
}
}
else args->fname = argv[optind];
+ if ( args->fai_fname ) update_from_fai(args);
if ( !args->samples_fname && !args->header_fname ) usage(args);
if ( !args->fname ) usage(args);
args->fp = hts_open(args->fname,"r");
- if ( !args->fp ) error("Failed to open: %s\n", args->fname);
+ if ( !args->fp ) error("Failed to read from %s\n", !strcmp("-",args->fname)?"standard input":args->fname);
args->type = *hts_get_format(args->fp);
if ( args->type.format==vcf )
else
reheader_bcf(args, args->type.compression==bgzf || args->type.compression==gzip);
+ if ( args->rm_tmpfile )
+ {
+ unlink(args->rm_tmpfile);
+ free(args->rm_tmpfile);
+ }
free(args);
return 0;
}
#include <inttypes.h>
#include <fcntl.h>
#include <math.h>
+#ifdef _WIN32
+#include <windows.h>
+#endif
#include <htslib/vcf.h>
#include <htslib/bgzf.h>
#include <htslib/tbx.h> // for hts_get_bgzfp()
#include <htslib/kseq.h>
#include <htslib/thread_pool.h>
+#include <htslib/faidx.h>
+#include <htslib/khash_str2int.h>
#include "bcftools.h"
#include "khash_str2str.h"
typedef struct _args_t
{
char **argv, *fname, *samples_fname, *header_fname, *output_fname;
+ char *fai_fname, *rm_tmpfile;
htsFile *fp;
htsFormat type;
htsThreadPool *threads;
}
args_t;
+static inline int is_escaped(const char *min, const char *str)
+{
+ int n = 0;
+ while ( --str>=min && *str=='\\' ) n++;
+ return n%2;
+}
+static char *copy_and_update_contig_line(faidx_t *fai, char *line, void *chr_seen, kstring_t *dst)
+{
+ kstring_t key = {0,0,0}, val = {0,0,0}, tmp = {0,0,0};
+ char *chr_name = NULL, *p, *q = line + 9; // skip ##contig=
+ char *end = q;
+ int nopen = 1, chr_len = 0;
+ while ( *end && *end!='\n' ) end++;
+ while ( *q && *q!='\n' && nopen>0 )
+ {
+ p = ++q;
+ while ( *q && (*q==' ' || *q=='\t') ) { p++; q++; }
+ // ^[A-Za-z_][0-9A-Za-z_.]*$
+ if (p==q && *q && (isalpha(*q) || *q=='_'))
+ {
+ q++;
+ while ( *q && (isalnum(*q) || *q=='_' || *q=='.') ) q++;
+ }
+ int n = q-p;
+ int m = 0;
+ while ( *q && (*q==' ' || *q=='\t') ) { q++; m++; }
+ if ( *q!='=' || !n )
+ {
+ char *x = q;
+ while ( *x && *x!='\n' ) x++;
+ *x = '\0';
+ error("Could not parse the line: %s [%s][%s]\n", line,p,q);
+ }
+ key.l = 0;
+ kputsn(p,q-p-m,&key);
+ p = ++q;
+ while ( *q && (*q==' ' || *q=='\t') ) { p++; q++; }
+ int quoted = *p=='"' ? 1 : 0;
+ if ( quoted ) p++, q++;
+ while ( *q && *q != '\n' )
+ {
+ if ( quoted ) { if ( *q=='"' && !is_escaped(p,q) ) break; }
+ else
+ {
+ if ( *q=='<' ) nopen++;
+ if ( *q=='>' ) nopen--;
+ if ( !nopen ) break;
+ if ( *q==',' && nopen==1 ) break;
+ }
+ q++;
+ }
+ char *r = q;
+ while ( r > p && r[-1] == ' ' ) r--;
+ val.l = 0;
+ kputsn(p,r-p,&val);
+ if ( quoted && *q=='"' ) q++;
+ if ( *q=='>' ) { nopen--; q++; }
+ if ( !strcmp("length",key.s) ) continue;
+ if ( !strcmp("ID",key.s) )
+ {
+ if ( khash_str2int_has_key(chr_seen,val.s) ) continue;
+ chr_len = faidx_seq_len(fai, val.s);
+ if ( chr_len==-1 )
+ {
+ free(val.s); free(key.s); free(tmp.s);
+ return end; // the sequence is not in fai, remove
+ }
+ chr_name = strdup(val.s);
+ khash_str2int_inc(chr_seen, chr_name);
+ continue;
+ }
+ kputc(',',&tmp);
+ kputs(key.s,&tmp);
+ kputc('=',&tmp);
+ if ( quoted ) kputc('"',&tmp);
+ kputs(val.s,&tmp);
+ if ( quoted ) kputc('"',&tmp);
+ }
+ if ( !chr_name ) return end;
+ ksprintf(dst,"##contig=<ID=%s,length=%d%s>",chr_name,chr_len,tmp.l ? tmp.s : "");
+ free(key.s); free(val.s); free(tmp.s);
+ return q;
+}
+static void update_from_fai(args_t *args)
+{
+ if ( !strcmp("-",args->fname) )
+ error("Cannot use the --fai option when reading from standard input.\n");
+
+ faidx_t *fai = fai_load3(args->fai_fname,args->fai_fname,NULL,FAI_FASTA);
+ if ( !fai ) error("Could not parse %s\n", args->fai_fname);
+#ifdef _WIN32
+ char tmp_path[MAX_PATH];
+ int ret = GetTempPath(MAX_PATH, tmp_path);
+ if (!ret || ret > MAX_PATH)
+ error("Could not get the path to the temporary folder\n");
+ if (strlen(tmp_path) + strlen("/bcftools-fai-header-XXXXXX") >= MAX_PATH)
+ error("Full path to the temporary folder is too long\n");
+ strcat(tmp_path, "/bcftools-fai-header-XXXXXX");
+ args->rm_tmpfile = strdup(tmp_path);
+#else
+ args->rm_tmpfile = strdup("/tmp/bcftools-fai-header-XXXXXX");
+#endif
+ int fd = mkstemp(args->rm_tmpfile);
+ if ( fd<0 ) error("Could not open a temporary file for writing: %s\n", args->rm_tmpfile);
+
+ // get a template header: either from the original VCF or from --header
+ char *ori_hdr_fname = args->header_fname ? args->header_fname : args->fname;
+ htsFile *fp = hts_open(ori_hdr_fname,"r");
+ if ( !fp ) error("Failed to open: %s\n", ori_hdr_fname);
+ bcf_hdr_t *hdr = bcf_hdr_read(fp);
+ if ( !hdr ) error("Failed to read the header: %s\n", ori_hdr_fname);
+ hts_close(fp); // no need to check the return status here
+
+ // put the header in a text buffer
+ kstring_t hdr_txt_ori = {0,0,0}, hdr_txt_new = {0,0,0};
+ bcf_hdr_format(hdr, 0, &hdr_txt_ori);
+ bcf_hdr_destroy(hdr);
+
+ // update the existing contig lines and remove lines not present in the fai file
+ void *chr_seen = khash_str2int_init();
+ char *tmp, *beg = hdr_txt_ori.s;
+ while ( beg && *beg )
+ {
+ tmp = strstr(beg, "\n##contig=<");
+ if ( !tmp ) break;
+ kputsn(beg, tmp-beg+1, &hdr_txt_new);
+ size_t l_prev = hdr_txt_new.l;
+ beg = copy_and_update_contig_line(fai,tmp+1,chr_seen, &hdr_txt_new);
+ if ( l_prev==hdr_txt_new.l ) hdr_txt_new.l--; // nothing was added, remove the newline
+ }
+ if ( !beg || !(tmp=strstr(beg,"\n#CHROM")) ) error("Failed to parse the header, #CHROM not found\n");
+ kputsn(beg, tmp-beg+1, &hdr_txt_new);
+
+ // add any new contig lines
+ int i, n = faidx_nseq(fai);
+ for (i=0; i<n; i++)
+ {
+ if ( khash_str2int_has_key(chr_seen,faidx_iseq(fai,i)) ) continue;
+ ksprintf(&hdr_txt_new,"##contig=<ID=%s,length=%d>\n",faidx_iseq(fai,i),faidx_seq_len(fai,faidx_iseq(fai,i)));
+ }
+ kputs(tmp+1,&hdr_txt_new);
+
+ if ( write(fd, hdr_txt_new.s, hdr_txt_new.l)!=hdr_txt_new.l ) error("Failed to write %zu bytes to %s\n", hdr_txt_new.l,args->rm_tmpfile);
+ if ( close(fd)!=0 ) error("Failed to close %s\n", args->rm_tmpfile);
+ args->header_fname = args->rm_tmpfile;
+
+ free(hdr_txt_ori.s);
+ free(hdr_txt_new.s);
+ fai_destroy(fai);
+ khash_str2int_destroy_free(chr_seen);
+}
+
static void read_header_file(char *fname, kstring_t *hdr)
{
kstring_t tmp = {0,0,0};
kputc('\n',&fp->line);
if ( write(out, fp->line.s, fp->line.l)!=fp->line.l ) error("Failed to write %"PRIu64" bytes\n", (uint64_t)fp->line.l);
}
- hts_close(fp);
- close(out);
+ if ( hts_close(fp)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fname);
+ if ( close(out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
}
static bcf_hdr_t *strip_header(bcf_hdr_t *src, bcf_hdr_t *dst)
if ( j>=0 )
{
j = atoi(src_hrec->vals[j]);
- hrec_add_idx(tmp, j);
+ if (hrec_add_idx(tmp, j) < 0)
+ error_errno("[%s] Failed to add IDX header", __func__);
}
bcf_hdr_add_hrec(out, tmp);
}
}
- bcf_hdr_sync(out);
+ if (bcf_hdr_sync(out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
for (i=0; i<dst->nhrec; i++)
{
// finally add new structured fields
if ( args->n_threads > 0 )
{
- args->threads = calloc(1, sizeof(*args->threads));
+ args->threads = (htsThreadPool *) calloc(1, sizeof(htsThreadPool));
if ( !args->threads ) error("Could not allocate memory\n");
if ( !(args->threads->pool = hts_tpool_init(args->n_threads)) ) error("Could not initialize threading\n");
- BGZF *bgzf = hts_get_bgzfp(fp);
- if ( bgzf ) bgzf_thread_pool(bgzf, args->threads->pool, args->threads->qsize);
+ hts_set_thread_pool(fp, args->threads);
}
bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to read the header: %s\n", args->fname);
htsFile *fp_out = hts_open(args->output_fname ? args->output_fname : "-",is_compressed ? "wb" : "wbu");
if ( !fp_out ) error("%s: %s\n", args->output_fname ? args->output_fname : "-", strerror(errno));
if ( args->threads )
- {
- BGZF *bgzf = hts_get_bgzfp(fp_out);
- if ( bgzf ) bgzf_thread_pool(bgzf, args->threads->pool, args->threads->qsize);
- }
- bcf_hdr_write(fp_out, hdr_out);
+ hts_set_thread_pool(fp_out, args->threads);
+ if ( bcf_hdr_write(fp_out, hdr_out)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname ? args->output_fname : "standard output");
bcf1_t *rec = bcf_init();
while ( bcf_read(fp, hdr, rec)==0 )
if ( i!=rec->n_fmt )
error("The FORMAT tag is not defined: \"%s\"\n", bcf_hdr_int2id(hdr,BCF_DT_ID,rec->d.fmt[i].id));
- bcf_write(fp_out,hdr_out,rec);
+ if ( bcf_write(fp_out,hdr_out,rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname ? args->output_fname : "standard output");
}
bcf_destroy(rec);
free(htxt.s);
- hts_close(fp_out);
- hts_close(fp);
+ if ( hts_close(fp_out)!=0 ) error("[%s] Error: failed to close the file %s\n",__func__,args->output_fname ? args->output_fname : "standard output");
+ if ( hts_close(fp)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fname);
bcf_hdr_destroy(hdr_out);
bcf_hdr_destroy(hdr);
if ( args->threads )
fprintf(bcftools_stderr, "Usage: bcftools reheader [OPTIONS] <in.vcf.gz>\n");
fprintf(bcftools_stderr, "\n");
fprintf(bcftools_stderr, "Options:\n");
+ fprintf(bcftools_stderr, " -f, --fai <file> update sequences and their lengths from the .fai file\n");
fprintf(bcftools_stderr, " -h, --header <file> new header\n");
fprintf(bcftools_stderr, " -o, --output <file> write output to a file [standard output]\n");
fprintf(bcftools_stderr, " -s, --samples <file> new sample names\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra compression threads (BCF only) [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads (BCF only) [0]\n");
+ fprintf(bcftools_stderr, "\n");
+ fprintf(bcftools_stderr, "Example:\n");
+ fprintf(bcftools_stderr, " # Write out the header to be modified\n");
+ fprintf(bcftools_stderr, " bcftools view -h old.bcf > header.txt\n");
+ fprintf(bcftools_stderr, "\n");
+ fprintf(bcftools_stderr, " # Edit the header using your favorite text editor\n");
+ fprintf(bcftools_stderr, " vi header.txt\n");
+ fprintf(bcftools_stderr, "\n");
+ fprintf(bcftools_stderr, " # Reheader the file\n");
+ fprintf(bcftools_stderr, " bcftools reheader -h header.txt -o new.bcf old.bcf\n");
fprintf(bcftools_stderr, "\n");
exit(1);
}
static struct option loptions[] =
{
+ {"fai",1,0,'f'},
{"output",1,0,'o'},
{"header",1,0,'h'},
{"samples",1,0,'s'},
{"threads",1,NULL,1},
{0,0,0,0}
};
- while ((c = getopt_long(argc, argv, "s:h:o:",loptions,NULL)) >= 0)
+ while ((c = getopt_long(argc, argv, "s:h:o:f:",loptions,NULL)) >= 0)
{
switch (c)
{
case 1 : args->n_threads = strtol(optarg, 0, 0); break;
+ case 'f': args->fai_fname = optarg; break;
case 'o': args->output_fname = optarg; break;
case 's': args->samples_fname = optarg; break;
case 'h': args->header_fname = optarg; break;
- case '?': usage(args);
+ case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
}
}
else args->fname = argv[optind];
+ if ( args->fai_fname ) update_from_fai(args);
if ( !args->samples_fname && !args->header_fname ) usage(args);
if ( !args->fname ) usage(args);
args->fp = hts_open(args->fname,"r");
- if ( !args->fp ) error("Failed to open: %s\n", args->fname);
+ if ( !args->fp ) error("Failed to read from %s\n", !strcmp("-",args->fname)?"standard input":args->fname);
args->type = *hts_get_format(args->fp);
if ( args->type.format==vcf )
else
reheader_bcf(args, args->type.compression==bgzf || args->type.compression==gzip);
+ if ( args->rm_tmpfile )
+ {
+ unlink(args->rm_tmpfile);
+ free(args->rm_tmpfile);
+ }
free(args);
return 0;
}
THE SOFTWARE.
*/
+#include <ctype.h>
#include "bcftools.h"
#include "smpl_ilist.h"
void smpl_ilist_destroy(smpl_ilist_t *smpl)
{
+ int i;
+ if ( smpl->pair )
+ {
+ for (i=0; i<smpl->n; i++) free(smpl->pair[i]);
+ free(smpl->pair);
+ }
free(smpl->idx);
free(smpl);
}
+static inline int is_space_or_escaped(const char *min, const char *str)
+{
+ if ( !isspace(*str) ) return 0;
+ int n = 0;
+ while ( --str>=min && *str=='\\' ) n++;
+ return n%2 ? 0 : 1;
+}
smpl_ilist_t *smpl_ilist_init(bcf_hdr_t *hdr, char *sample_list, int is_file, int flags)
{
smpl_ilist_t *smpl = (smpl_ilist_t*) calloc(1,sizeof(smpl_ilist_t));
return smpl;
}
+ int negate = sample_list[0]=='^' ? 1 : 0;
int nlist;
- char **list = hts_readlist(sample_list[0]=='^'?sample_list+1:sample_list, is_file, &nlist);
+ char **list = hts_readlist(negate?sample_list+1:sample_list, is_file, &nlist);
if ( !list ) error("Could not parse %s\n", sample_list);
// preserve the VCF order
int *tmp = (int*)calloc(bcf_hdr_nsamples(hdr),sizeof(int));
+ char **pair = NULL;
for (i=0; i<nlist; i++)
{
- int idx = bcf_hdr_id2int(hdr, BCF_DT_SAMPLE, list[i]);
- if ( idx>=0 )
+ char *smpl1 = list[i];
+ char *smpl2 = NULL;
+
+ char *ptr = list[i];
+ while ( *ptr && !is_space_or_escaped(list[i], ptr) ) ptr++;
+ if ( *ptr )
+ {
+ *ptr = 0;
+ smpl2 = ptr+1;
+ }
+
+ char *smpl_name = flags&SMPL_PAIR2 && smpl2 ? smpl2 : smpl1;
+ int idx = bcf_hdr_id2int(hdr, BCF_DT_SAMPLE, smpl_name);
+ if ( idx<0 )
{
- tmp[idx] = 1;
- smpl->n++;
+ if ( !(flags&SMPL_STRICT) )
+ {
+ if ( flags&SMPL_VERBOSE ) fprintf(stderr,"No such sample: \"%s\"\n",smpl_name);
+ continue;
+ }
+ error("No such sample: \"%s\"\n", smpl_name);
}
- else if ( flags&SMPL_STRICT )
- error("No such sample: %s\n", list[i]);
+
+ tmp[idx] = 1;
+ if ( smpl2 )
+ {
+ if ( !pair ) pair = (char**)calloc(bcf_hdr_nsamples(hdr),sizeof(char*));
+ if ( flags&SMPL_PAIR2 ) pair[idx] = strdup(smpl1);
+ else if ( flags&SMPL_PAIR1 ) pair[idx] = strdup(smpl2);
+ }
+ smpl->n++;
}
- if ( sample_list[0]=='^' ) smpl->n = bcf_hdr_nsamples(hdr) - smpl->n;
+ if ( negate ) smpl->n = bcf_hdr_nsamples(hdr) - smpl->n;
smpl->idx = (int*) malloc(sizeof(int)*smpl->n);
int j = 0;
- if ( sample_list[0]!='^' )
+ if ( !negate )
{
+ if ( pair ) smpl->pair = (char**) calloc(smpl->n,sizeof(char*));
for (i=0; i<bcf_hdr_nsamples(hdr); i++)
- if ( tmp[i] ) smpl->idx[j++] = i;
+ {
+ if ( !tmp[i] ) continue;
+ smpl->idx[j] = i;
+ if ( pair && pair[i] ) smpl->pair[j] = pair[i];
+ j++;
+ }
}
else
{
}
free(tmp);
+ free(pair);
for (i=0; i<nlist; i++) free(list[i]);
free(list);
THE SOFTWARE.
*/
+#include <ctype.h>
#include "bcftools.h"
#include "smpl_ilist.h"
void smpl_ilist_destroy(smpl_ilist_t *smpl)
{
+ int i;
+ if ( smpl->pair )
+ {
+ for (i=0; i<smpl->n; i++) free(smpl->pair[i]);
+ free(smpl->pair);
+ }
free(smpl->idx);
free(smpl);
}
+static inline int is_space_or_escaped(const char *min, const char *str)
+{
+ if ( !isspace(*str) ) return 0;
+ int n = 0;
+ while ( --str>=min && *str=='\\' ) n++;
+ return n%2 ? 0 : 1;
+}
smpl_ilist_t *smpl_ilist_init(bcf_hdr_t *hdr, char *sample_list, int is_file, int flags)
{
smpl_ilist_t *smpl = (smpl_ilist_t*) calloc(1,sizeof(smpl_ilist_t));
return smpl;
}
+ int negate = sample_list[0]=='^' ? 1 : 0;
int nlist;
- char **list = hts_readlist(sample_list[0]=='^'?sample_list+1:sample_list, is_file, &nlist);
+ char **list = hts_readlist(negate?sample_list+1:sample_list, is_file, &nlist);
if ( !list ) error("Could not parse %s\n", sample_list);
// preserve the VCF order
int *tmp = (int*)calloc(bcf_hdr_nsamples(hdr),sizeof(int));
+ char **pair = NULL;
for (i=0; i<nlist; i++)
{
- int idx = bcf_hdr_id2int(hdr, BCF_DT_SAMPLE, list[i]);
- if ( idx>=0 )
+ char *smpl1 = list[i];
+ char *smpl2 = NULL;
+
+ char *ptr = list[i];
+ while ( *ptr && !is_space_or_escaped(list[i], ptr) ) ptr++;
+ if ( *ptr )
+ {
+ *ptr = 0;
+ smpl2 = ptr+1;
+ }
+
+ char *smpl_name = flags&SMPL_PAIR2 && smpl2 ? smpl2 : smpl1;
+ int idx = bcf_hdr_id2int(hdr, BCF_DT_SAMPLE, smpl_name);
+ if ( idx<0 )
{
- tmp[idx] = 1;
- smpl->n++;
+ if ( !(flags&SMPL_STRICT) )
+ {
+ if ( flags&SMPL_VERBOSE ) fprintf(bcftools_stderr,"No such sample: \"%s\"\n",smpl_name);
+ continue;
+ }
+ error("No such sample: \"%s\"\n", smpl_name);
}
- else if ( flags&SMPL_STRICT )
- error("No such sample: %s\n", list[i]);
+
+ tmp[idx] = 1;
+ if ( smpl2 )
+ {
+ if ( !pair ) pair = (char**)calloc(bcf_hdr_nsamples(hdr),sizeof(char*));
+ if ( flags&SMPL_PAIR2 ) pair[idx] = strdup(smpl1);
+ else if ( flags&SMPL_PAIR1 ) pair[idx] = strdup(smpl2);
+ }
+ smpl->n++;
}
- if ( sample_list[0]=='^' ) smpl->n = bcf_hdr_nsamples(hdr) - smpl->n;
+ if ( negate ) smpl->n = bcf_hdr_nsamples(hdr) - smpl->n;
smpl->idx = (int*) malloc(sizeof(int)*smpl->n);
int j = 0;
- if ( sample_list[0]!='^' )
+ if ( !negate )
{
+ if ( pair ) smpl->pair = (char**) calloc(smpl->n,sizeof(char*));
for (i=0; i<bcf_hdr_nsamples(hdr); i++)
- if ( tmp[i] ) smpl->idx[j++] = i;
+ {
+ if ( !tmp[i] ) continue;
+ smpl->idx[j] = i;
+ if ( pair && pair[i] ) smpl->pair[j] = pair[i];
+ j++;
+ }
}
else
{
}
free(tmp);
+ free(pair);
for (i=0; i<nlist; i++) free(list[i]);
free(list);
#include <htslib/vcf.h>
-#define SMPL_NONE 0 // flexible error recovery
-#define SMPL_STRICT 1 // samples must exist
+#define SMPL_NONE 0 // flexible error recovery
+#define SMPL_STRICT 1 // samples must exist
+#define SMPL_SINGLE 2 // single sample expected
+#define SMPL_PAIR1 4 // two samples expected, the first is from the bcf hdr
+#define SMPL_PAIR2 8 // two samples expected, the second is from the bcf hdr
+#define SMPL_VERBOSE 16 // print warnings
typedef struct
{
- int *idx; // index to bcf_hdr_t.samples
+ char **pair; // the other sample in the pair
+ int *idx; // index to bcf_hdr_t.samples; the first (SMPL_SINGLE|SMPL_PAIR1) or second sample (SMPL_PAIR2)
int n;
}
smpl_ilist_t;
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
+#include <strings.h>
#include <htslib/bgzf.h>
#include <htslib/tbx.h>
{
// auto-detect file type by file name
int l = strlen(argv[optind]);
- int strcasecmp(const char *s1, const char *s2);
if (l>=7 && strcasecmp(argv[optind]+l-7, ".gff.gz") == 0) conf = tbx_conf_gff;
else if (l>=7 && strcasecmp(argv[optind]+l-7, ".bed.gz") == 0) conf = tbx_conf_bed;
else if (l>=7 && strcasecmp(argv[optind]+l-7, ".sam.gz") == 0) conf = tbx_conf_sam;
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
+#include <strings.h>
#include <htslib/bgzf.h>
#include <htslib/tbx.h>
{
// auto-detect file type by file name
int l = strlen(argv[optind]);
- int strcasecmp(const char *s1, const char *s2);
if (l>=7 && strcasecmp(argv[optind]+l-7, ".gff.gz") == 0) conf = tbx_conf_gff;
else if (l>=7 && strcasecmp(argv[optind]+l-7, ".bed.gz") == 0) conf = tbx_conf_bed;
else if (l>=7 && strcasecmp(argv[optind]+l-7, ".sam.gz") == 0) conf = tbx_conf_sam;
+++ /dev/null
-/* test/test-rbuf.c -- rbuf_t test harness.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include "rbuf.h"
-
-void debug_print(rbuf_t *rbuf, int *dat)
-{
- int i;
- for (i=-1; rbuf_next(rbuf, &i); ) printf(" %2d", i);
- printf("\n");
- for (i=-1; rbuf_next(rbuf, &i); ) printf(" %2d", dat[i]);
- printf("\n");
-}
-
-int main(int argc, char **argv)
-{
- int i, j, *dat = (int*)calloc(10,sizeof(int));
- rbuf_t rbuf;
- rbuf_init(&rbuf,10);
-
- rbuf.f = 5; // force wrapping
- for (i=0; i<9; i++)
- {
- j = rbuf_append(&rbuf);
- dat[j] = i+1;
- }
- printf("Inserted 1-9 starting at offset 5:\n");
- debug_print(&rbuf, dat);
-
- i = rbuf_kth(&rbuf, 3);
- printf("4th is %d\n", dat[i]);
-
- printf("Deleting 1-2:\n");
- rbuf_shift_n(&rbuf, 2);
- debug_print(&rbuf, dat);
-
- printf("Prepending 0-8:\n");
- for (i=0; i<9; i++)
- {
- j = rbuf_prepend(&rbuf);
- dat[j] = i;
- }
- debug_print(&rbuf, dat);
-
- printf("Expanding:\n");
- rbuf_expand0(&rbuf,int,rbuf.n+1,dat);
- debug_print(&rbuf, dat);
-
- free(dat);
- return 0;
-}
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* test/test-rbuf.c -- rbuf_t test harness.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include "rbuf.h"
-
-void debug_print(rbuf_t *rbuf, int *dat)
-{
- int i;
- for (i=-1; rbuf_next(rbuf, &i); ) fprintf(bcftools_stdout, " %2d", i);
- fprintf(bcftools_stdout, "\n");
- for (i=-1; rbuf_next(rbuf, &i); ) fprintf(bcftools_stdout, " %2d", dat[i]);
- fprintf(bcftools_stdout, "\n");
-}
-
-int bcftools_test-rbuf_main(int argc, char **argv)
-{
- int i, j, *dat = (int*)calloc(10,sizeof(int));
- rbuf_t rbuf;
- rbuf_init(&rbuf,10);
-
- rbuf.f = 5; // force wrapping
- for (i=0; i<9; i++)
- {
- j = rbuf_append(&rbuf);
- dat[j] = i+1;
- }
- fprintf(bcftools_stdout, "Inserted 1-9 starting at offset 5:\n");
- debug_print(&rbuf, dat);
-
- i = rbuf_kth(&rbuf, 3);
- fprintf(bcftools_stdout, "4th is %d\n", dat[i]);
-
- fprintf(bcftools_stdout, "Deleting 1-2:\n");
- rbuf_shift_n(&rbuf, 2);
- debug_print(&rbuf, dat);
-
- fprintf(bcftools_stdout, "Prepending 0-8:\n");
- for (i=0; i<9; i++)
- {
- j = rbuf_prepend(&rbuf);
- dat[j] = i;
- }
- debug_print(&rbuf, dat);
-
- fprintf(bcftools_stdout, "Expanding:\n");
- rbuf_expand0(&rbuf,int,rbuf.n+1,dat);
- debug_print(&rbuf, dat);
-
- free(dat);
- return 0;
-}
-
+++ /dev/null
-/* test/test-regidx.c -- Regions index test harness.
-
- gcc -g -Wall -O0 -I. -I../htslib/ -L../htslib regidx.c -o test-regidx test-regidx.c -lhts
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <string.h>
-#include <getopt.h>
-#include <htslib/kstring.h>
-#include <time.h>
-#include "regidx.h"
-
-static int verbose = 0;
-
-void debug(const char *format, ...)
-{
- if ( verbose<2 ) return;
- va_list ap;
- va_start(ap, format);
- vfprintf(stderr, format, ap);
- va_end(ap);
-}
-void info(const char *format, ...)
-{
- if ( verbose<1 ) return;
- va_list ap;
- va_start(ap, format);
- vfprintf(stderr, format, ap);
- va_end(ap);
-}
-void error(const char *format, ...)
-{
- va_list ap;
- va_start(ap, format);
- vfprintf(stderr, format, ap);
- va_end(ap);
- exit(-1);
-}
-
-int custom_parse(const char *line, char **chr_beg, char **chr_end, uint32_t *beg, uint32_t *end, void *payload, void *usr)
-{
- // Use the standard parser for CHROM,FROM,TO
- int i, ret = regidx_parse_tab(line,chr_beg,chr_end,beg,end,NULL,NULL);
- if ( ret!=0 ) return ret;
-
- // Skip the fields that were parsed above
- char *ss = (char*) line;
- while ( *ss && isspace(*ss) ) ss++;
- for (i=0; i<3; i++)
- {
- while ( *ss && !isspace(*ss) ) ss++;
- if ( !*ss ) return -2; // wrong number of fields
- while ( *ss && isspace(*ss) ) ss++;
- }
- if ( !*ss ) return -2;
-
- // Parse the payload
- char *se = ss;
- while ( *se && !isspace(*se) ) se++;
- char **dat = (char**) payload;
- *dat = (char*) malloc(se-ss+1);
- memcpy(*dat,ss,se-ss+1);
- (*dat)[se-ss] = 0;
- return 0;
-}
-void custom_free(void *payload)
-{
- char **dat = (char**)payload;
- free(*dat);
-}
-
-void test_sequential_access(void)
-{
- // Init index with no file name, we will insert the regions manually
- regidx_t *idx = regidx_init(NULL,custom_parse,custom_free,sizeof(char*),NULL);
- if ( !idx ) error("init failed\n");
-
- // Insert regions
- kstring_t str = {0,0,0};
- int i, n = 10;
- for (i=0; i<n; i++)
- {
- int beg = 10*(i+1);
- str.l = 0;
- ksprintf(&str,"1\t%d\t%d\t%d",beg,beg,beg);
- if ( regidx_insert(idx,str.s)!=0 ) error("insert failed: %s\n",str.s);
- }
-
- // Test
- regitr_t *itr = regitr_init(idx);
- i = 0;
- while ( regitr_loop(itr) )
- {
- if ( itr->beg!=itr->end || itr->beg+1!=10*(i+1) ) error("listing failed, expected %d, found %d\n",10*(i+1),itr->beg+1);
- str.l = 0;
- ksprintf(&str,"%d",itr->beg+1);
- if ( strcmp(regitr_payload(itr,char*),str.s) ) error("listing failed, expected payload \"%s\", found \"%s\"\n",str.s,regitr_payload(itr,char*));
- i++;
- }
- if ( i!=n ) error("Expected %d regions, listed %d\n", n,i);
- debug("ok: listed %d regions\n", n);
-
- // Clean up
- regitr_destroy(itr);
- regidx_destroy(idx);
- free(str.s);
-}
-
-void test_custom_payload(void)
-{
- // Init index with no file name, we will insert the regions manually
- regidx_t *idx = regidx_init(NULL,custom_parse,custom_free,sizeof(char*),NULL);
- if ( !idx ) error("init failed\n");
-
- // Insert regions
- char *line;
- line = "1 10000000 10000000 1:10000000-10000000"; if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
- line = "1 20000000 20000001 1:20000000-20000001"; if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
- line = "1 20000002 20000002 1:20000002-20000002"; if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
- line = "1 30000000 30000000 1:30000000-30000000"; if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
-
- // Test
- regitr_t *itr = regitr_init(idx);
- int from, to;
-
- from = to = 10000000;
- if ( !regidx_overlap(idx,"1",from-1,to-1,itr) ) error("query failed: 1:%d-%d\n",from,to);
- if ( strcmp("1:10000000-10000000",regitr_payload(itr,char*)) ) error("query failed: 1:%d-%d vs %s\n", from,to,regitr_payload(itr,char*));
- if ( !regidx_overlap(idx,"1",from-2,to-1,itr) ) error("query failed: 1:%d-%d\n",from-1,to);
- if ( !regidx_overlap(idx,"1",from-2,to+3,itr) ) error("query failed: 1:%d-%d\n",from-1,to+2);
- if ( regidx_overlap(idx,"1",from-2,to-2,itr) ) error("query failed: 1:%d-%d\n",from-1,to-1);
-
- from = to = 20000000;
- if ( !regidx_overlap(idx,"1",from-1,to-1,itr) ) error("query failed: 1:%d-%d\n",from,to);
-
- from = to = 20000002;
- if ( !regidx_overlap(idx,"1",from-1,to-1,itr) ) error("query failed: 1:%d-%d\n",from,to);
-
- from = to = 30000000;
- if ( !regidx_overlap(idx,"1",from-1,to-1,itr) ) error("query failed: 1:%d-%d\n",from,to);
-
- // Clean up
- regitr_destroy(itr);
- regidx_destroy(idx);
-}
-
-void get_random_region(uint32_t min, uint32_t max, uint32_t *beg, uint32_t *end)
-{
- long int b = random(), e = random();
- *beg = min + (float)b * (max-min) / RAND_MAX;
- *end = *beg + (float)e * (max-*beg) / RAND_MAX;
-}
-
-void test_random(int nregs, uint32_t min, uint32_t max)
-{
- min--;
- max--;
-
- // Init index with no file name, we will insert the regions manually
- regidx_t *idx = regidx_init(NULL,custom_parse,custom_free,sizeof(char*),NULL);
- if ( !idx ) error("init failed\n");
-
- // Test region
- uint32_t beg,end;
- get_random_region(min,max,&beg,&end);
-
- // Insert regions
- int i, nexp = 0;
- kstring_t str = {0,0,0};
- for (i=0; i<nregs; i++)
- {
- uint32_t b,e;
- get_random_region(min,max,&b,&e);
- str.l = 0;
- ksprintf(&str,"1\t%"PRIu32"\t%"PRIu32"\t1:%"PRIu32"-%"PRIu32"",b+1,e+1,b+1,e+1);
- if ( regidx_insert(idx,str.s)!=0 ) error("insert failed: %s\n", str.s);
- if ( e>=beg && b<=end ) nexp++;
- }
-
- // Test
- regitr_t *itr = regitr_init(idx);
- int nhit = 0, ret = regidx_overlap(idx,"1",beg,end,itr);
- if ( nexp && !ret ) error("query failed, expected %d overlap(s), found none: %d-%d\n", nexp,beg+1,end+1);
- if ( !nexp && ret ) error("query failed, expected no overlaps, found some: %d-%d\n", beg+1,end+1);
- while ( ret && regitr_overlap(itr) )
- {
- str.l = 0;
- ksprintf(&str,"1:%"PRIu32"-%"PRIu32"",itr->beg+1,itr->end+1);
- if ( strcmp(str.s,regitr_payload(itr,char*)) )
- error("query failed, incorrect payload: %s vs %s (%d-%d)\n",str.s,regitr_payload(itr,char*),beg+1,end+1);
- if ( itr->beg > end || itr->end < beg )
- error("query failed, incorrect hit: %d-%d vs %d-%d, payload %s\n", beg+1,end+1,itr->beg+1,itr->end+1,regitr_payload(itr,char*));
- nhit++;
- }
- if ( nexp!=nhit ) error("query failed, expected %d overlap(s), found %d: %d-%d\n",nexp,nhit,beg+1,end+1);
- debug("ok: found %d overlaps\n", nexp);
-
- // Clean up
- regitr_destroy(itr);
- regidx_destroy(idx);
- free(str.s);
-}
-
-void create_line_bed(char *line, char *chr, int start, int end)
-{
- sprintf(line,"%s\t%d\t%d\n",chr,start-1,end);
-}
-void create_line_tab(char *line, char *chr, int start, int end)
-{
- sprintf(line,"%s\t%d\t%d\n",chr,start,end);
-}
-void create_line_reg(char *line, char *chr, int start, int end)
-{
- sprintf(line,"%s:%d-%d\n",chr,start,end);
-}
-
-typedef void (*set_line_f)(char *line, char *chr, int start, int end);
-
-void test(set_line_f set_line, regidx_parse_f parse)
-{
- regidx_t *idx = regidx_init(NULL,parse,NULL,0,NULL);
- if ( !idx ) error("init failed\n");
-
- char line[250], *chr = "1";
- int i, n = 10, start, end, nhit;
- for (i=1; i<n; i++)
- {
- start = end = 10*i;
- set_line(line,chr,start,end);
- debug("insert: %s", line);
- if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
-
- start = end = 10*i + 1;
- set_line(line,chr,start,end);
- debug("insert: %s", line);
- if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
- }
-
- regitr_t *itr = regitr_init(idx);
- for (i=1; i<n; i++)
- {
- // no hit
- start = end = 10*i - 1;
- if ( regidx_overlap(idx,chr,start-1,end-1,itr) ) error("query failed, there should be no hit: %s:%d-%d\n",chr,start,end);
- debug("ok: no overlap found for %s:%d-%d\n",chr,start,end);
-
-
- // one hit
- start = end = 10*i;
- if ( !regidx_overlap(idx,chr,start-1,end-1,itr) ) error("query failed, there should be a hit: %s:%d-%d\n",chr,start,end);
- debug("ok: overlap(s) found for %s:%d-%d\n",chr,start,end);
- nhit = 0;
- while ( regitr_overlap(itr) )
- {
- if ( itr->beg > end-1 || itr->end < start-1 ) error("query failed, incorrect region: %d-%d for %d-%d\n",itr->beg+1,itr->end+1,start,end);
- debug("\t %d-%d\n",itr->beg+1,itr->end+1);
- nhit++;
- }
- if ( nhit!=1 ) error("query failed, expected one hit, found %d: %s:%d-%d\n",nhit,chr,start,end);
-
-
- // one hit
- start = end = 10*i+1;
- if ( !regidx_overlap(idx,chr,start-1,end-1,itr) ) error("query failed, there should be a hit: %s:%d-%d\n",chr,start,end);
- debug("ok: overlap(s) found for %s:%d-%d\n",chr,start,end);
- nhit = 0;
- while ( regitr_overlap(itr) )
- {
- if ( itr->beg > end-1 || itr->end < start-1 ) error("query failed, incorrect region: %d-%d for %d-%d\n",itr->beg+1,itr->end+1,start,end);
- debug("\t %d-%d\n",itr->beg+1,itr->end+1);
- nhit++;
- }
- if ( nhit!=1 ) error("query failed, expected one hit, found %d: %s:%d-%d\n",nhit,chr,start,end);
-
-
- // two hits
- start = 10*i; end = start+1;
- if ( !regidx_overlap(idx,chr,start-1,end-1,itr) ) error("query failed, there should be a hit: %s:%d-%d\n",chr,start,end);
- debug("ok: overlap(s) found for %s:%d-%d\n",chr,start,end);
- nhit = 0;
- while ( regitr_overlap(itr) )
- {
- if ( itr->beg > end-1 || itr->end < start-1 ) error("query failed, incorrect region: %d-%d for %d-%d\n",itr->beg+1,itr->end+1,start,end);
- debug("\t %d-%d\n",itr->beg+1,itr->end+1);
- nhit++;
- }
- if ( nhit!=2 ) error("query failed, expected two hits, found %d: %s:%d-%d\n",nhit,chr,start,end);
-
- }
- regitr_destroy(itr);
- regidx_destroy(idx);
-}
-
-static void usage(void)
-{
- fprintf(stderr, "Usage: test-regidx [OPTIONS]\n");
- fprintf(stderr, "Options:\n");
- fprintf(stderr, " -h, --help this help message\n");
- fprintf(stderr, " -s, --seed <int> random seed\n");
- fprintf(stderr, " -v, --verbose increase verbosity by giving multiple times\n");
-
- exit(1);
-}
-
-int main(int argc, char **argv)
-{
- static struct option loptions[] =
- {
- {"help",0,0,'h'},
- {"verbose",0,0,'v'},
- {"seed",1,0,'s'},
- {0,0,0,0}
- };
- int c;
- int seed = (int)time(NULL);
- while ((c = getopt_long(argc, argv, "hvs:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 's': seed = atoi(optarg); break;
- case 'v': verbose++; break;
- default: usage(); break;
- }
- }
-
- info("Testing sequential access\n");
- test_sequential_access();
-
- info("Testing TAB\n");
- test(create_line_tab,regidx_parse_tab);
-
- info("Testing REG\n");
- test(create_line_reg,regidx_parse_reg);
-
- info("Testing BED\n");
- test(create_line_bed,regidx_parse_bed);
-
- info("Testing custom payload\n");
- test_custom_payload();
-
- int i, ntest = 1000, nreg = 50;
- srandom(seed);
- info("%d randomized tests, %d regions per test. Random seed is %d\n", ntest,nreg,seed);
- for (i=0; i<ntest; i++) test_random(nreg,1,1000);
-
- return 0;
-}
-
-
+++ /dev/null
-#include "bcftools.pysam.h"
-
-/* test/test-regidx.c -- Regions index test harness.
-
- gcc -g -Wall -O0 -I. -I../htslib/ -L../htslib regidx.c -o test-regidx test-regidx.c -lhts
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
-*/
-
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <string.h>
-#include <getopt.h>
-#include <htslib/kstring.h>
-#include <time.h>
-#include "regidx.h"
-
-static int verbose = 0;
-
-void debug(const char *format, ...)
-{
- if ( verbose<2 ) return;
- va_list ap;
- va_start(ap, format);
- vfprintf(bcftools_stderr, format, ap);
- va_end(ap);
-}
-void info(const char *format, ...)
-{
- if ( verbose<1 ) return;
- va_list ap;
- va_start(ap, format);
- vfprintf(bcftools_stderr, format, ap);
- va_end(ap);
-}
-void error(const char *format, ...)
-{
- va_list ap;
- va_start(ap, format);
- vfprintf(bcftools_stderr, format, ap);
- va_end(ap);
- exit(-1);
-}
-
-int custom_parse(const char *line, char **chr_beg, char **chr_end, uint32_t *beg, uint32_t *end, void *payload, void *usr)
-{
- // Use the standard parser for CHROM,FROM,TO
- int i, ret = regidx_parse_tab(line,chr_beg,chr_end,beg,end,NULL,NULL);
- if ( ret!=0 ) return ret;
-
- // Skip the fields that were parsed above
- char *ss = (char*) line;
- while ( *ss && isspace(*ss) ) ss++;
- for (i=0; i<3; i++)
- {
- while ( *ss && !isspace(*ss) ) ss++;
- if ( !*ss ) return -2; // wrong number of fields
- while ( *ss && isspace(*ss) ) ss++;
- }
- if ( !*ss ) return -2;
-
- // Parse the payload
- char *se = ss;
- while ( *se && !isspace(*se) ) se++;
- char **dat = (char**) payload;
- *dat = (char*) malloc(se-ss+1);
- memcpy(*dat,ss,se-ss+1);
- (*dat)[se-ss] = 0;
- return 0;
-}
-void custom_free(void *payload)
-{
- char **dat = (char**)payload;
- free(*dat);
-}
-
-void test_sequential_access(void)
-{
- // Init index with no file name, we will insert the regions manually
- regidx_t *idx = regidx_init(NULL,custom_parse,custom_free,sizeof(char*),NULL);
- if ( !idx ) error("init failed\n");
-
- // Insert regions
- kstring_t str = {0,0,0};
- int i, n = 10;
- for (i=0; i<n; i++)
- {
- int beg = 10*(i+1);
- str.l = 0;
- ksprintf(&str,"1\t%d\t%d\t%d",beg,beg,beg);
- if ( regidx_insert(idx,str.s)!=0 ) error("insert failed: %s\n",str.s);
- }
-
- // Test
- regitr_t *itr = regitr_init(idx);
- i = 0;
- while ( regitr_loop(itr) )
- {
- if ( itr->beg!=itr->end || itr->beg+1!=10*(i+1) ) error("listing failed, expected %d, found %d\n",10*(i+1),itr->beg+1);
- str.l = 0;
- ksprintf(&str,"%d",itr->beg+1);
- if ( strcmp(regitr_payload(itr,char*),str.s) ) error("listing failed, expected payload \"%s\", found \"%s\"\n",str.s,regitr_payload(itr,char*));
- i++;
- }
- if ( i!=n ) error("Expected %d regions, listed %d\n", n,i);
- debug("ok: listed %d regions\n", n);
-
- // Clean up
- regitr_destroy(itr);
- regidx_destroy(idx);
- free(str.s);
-}
-
-void test_custom_payload(void)
-{
- // Init index with no file name, we will insert the regions manually
- regidx_t *idx = regidx_init(NULL,custom_parse,custom_free,sizeof(char*),NULL);
- if ( !idx ) error("init failed\n");
-
- // Insert regions
- char *line;
- line = "1 10000000 10000000 1:10000000-10000000"; if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
- line = "1 20000000 20000001 1:20000000-20000001"; if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
- line = "1 20000002 20000002 1:20000002-20000002"; if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
- line = "1 30000000 30000000 1:30000000-30000000"; if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
-
- // Test
- regitr_t *itr = regitr_init(idx);
- int from, to;
-
- from = to = 10000000;
- if ( !regidx_overlap(idx,"1",from-1,to-1,itr) ) error("query failed: 1:%d-%d\n",from,to);
- if ( strcmp("1:10000000-10000000",regitr_payload(itr,char*)) ) error("query failed: 1:%d-%d vs %s\n", from,to,regitr_payload(itr,char*));
- if ( !regidx_overlap(idx,"1",from-2,to-1,itr) ) error("query failed: 1:%d-%d\n",from-1,to);
- if ( !regidx_overlap(idx,"1",from-2,to+3,itr) ) error("query failed: 1:%d-%d\n",from-1,to+2);
- if ( regidx_overlap(idx,"1",from-2,to-2,itr) ) error("query failed: 1:%d-%d\n",from-1,to-1);
-
- from = to = 20000000;
- if ( !regidx_overlap(idx,"1",from-1,to-1,itr) ) error("query failed: 1:%d-%d\n",from,to);
-
- from = to = 20000002;
- if ( !regidx_overlap(idx,"1",from-1,to-1,itr) ) error("query failed: 1:%d-%d\n",from,to);
-
- from = to = 30000000;
- if ( !regidx_overlap(idx,"1",from-1,to-1,itr) ) error("query failed: 1:%d-%d\n",from,to);
-
- // Clean up
- regitr_destroy(itr);
- regidx_destroy(idx);
-}
-
-void get_random_region(uint32_t min, uint32_t max, uint32_t *beg, uint32_t *end)
-{
- long int b = random(), e = random();
- *beg = min + (float)b * (max-min) / RAND_MAX;
- *end = *beg + (float)e * (max-*beg) / RAND_MAX;
-}
-
-void test_random(int nregs, uint32_t min, uint32_t max)
-{
- min--;
- max--;
-
- // Init index with no file name, we will insert the regions manually
- regidx_t *idx = regidx_init(NULL,custom_parse,custom_free,sizeof(char*),NULL);
- if ( !idx ) error("init failed\n");
-
- // Test region
- uint32_t beg,end;
- get_random_region(min,max,&beg,&end);
-
- // Insert regions
- int i, nexp = 0;
- kstring_t str = {0,0,0};
- for (i=0; i<nregs; i++)
- {
- uint32_t b,e;
- get_random_region(min,max,&b,&e);
- str.l = 0;
- ksprintf(&str,"1\t%"PRIu32"\t%"PRIu32"\t1:%"PRIu32"-%"PRIu32"",b+1,e+1,b+1,e+1);
- if ( regidx_insert(idx,str.s)!=0 ) error("insert failed: %s\n", str.s);
- if ( e>=beg && b<=end ) nexp++;
- }
-
- // Test
- regitr_t *itr = regitr_init(idx);
- int nhit = 0, ret = regidx_overlap(idx,"1",beg,end,itr);
- if ( nexp && !ret ) error("query failed, expected %d overlap(s), found none: %d-%d\n", nexp,beg+1,end+1);
- if ( !nexp && ret ) error("query failed, expected no overlaps, found some: %d-%d\n", beg+1,end+1);
- while ( ret && regitr_overlap(itr) )
- {
- str.l = 0;
- ksprintf(&str,"1:%"PRIu32"-%"PRIu32"",itr->beg+1,itr->end+1);
- if ( strcmp(str.s,regitr_payload(itr,char*)) )
- error("query failed, incorrect payload: %s vs %s (%d-%d)\n",str.s,regitr_payload(itr,char*),beg+1,end+1);
- if ( itr->beg > end || itr->end < beg )
- error("query failed, incorrect hit: %d-%d vs %d-%d, payload %s\n", beg+1,end+1,itr->beg+1,itr->end+1,regitr_payload(itr,char*));
- nhit++;
- }
- if ( nexp!=nhit ) error("query failed, expected %d overlap(s), found %d: %d-%d\n",nexp,nhit,beg+1,end+1);
- debug("ok: found %d overlaps\n", nexp);
-
- // Clean up
- regitr_destroy(itr);
- regidx_destroy(idx);
- free(str.s);
-}
-
-void create_line_bed(char *line, char *chr, int start, int end)
-{
- sprintf(line,"%s\t%d\t%d\n",chr,start-1,end);
-}
-void create_line_tab(char *line, char *chr, int start, int end)
-{
- sprintf(line,"%s\t%d\t%d\n",chr,start,end);
-}
-void create_line_reg(char *line, char *chr, int start, int end)
-{
- sprintf(line,"%s:%d-%d\n",chr,start,end);
-}
-
-typedef void (*set_line_f)(char *line, char *chr, int start, int end);
-
-void test(set_line_f set_line, regidx_parse_f parse)
-{
- regidx_t *idx = regidx_init(NULL,parse,NULL,0,NULL);
- if ( !idx ) error("init failed\n");
-
- char line[250], *chr = "1";
- int i, n = 10, start, end, nhit;
- for (i=1; i<n; i++)
- {
- start = end = 10*i;
- set_line(line,chr,start,end);
- debug("insert: %s", line);
- if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
-
- start = end = 10*i + 1;
- set_line(line,chr,start,end);
- debug("insert: %s", line);
- if ( regidx_insert(idx,line)!=0 ) error("insert failed: %s\n", line);
- }
-
- regitr_t *itr = regitr_init(idx);
- for (i=1; i<n; i++)
- {
- // no hit
- start = end = 10*i - 1;
- if ( regidx_overlap(idx,chr,start-1,end-1,itr) ) error("query failed, there should be no hit: %s:%d-%d\n",chr,start,end);
- debug("ok: no overlap found for %s:%d-%d\n",chr,start,end);
-
-
- // one hit
- start = end = 10*i;
- if ( !regidx_overlap(idx,chr,start-1,end-1,itr) ) error("query failed, there should be a hit: %s:%d-%d\n",chr,start,end);
- debug("ok: overlap(s) found for %s:%d-%d\n",chr,start,end);
- nhit = 0;
- while ( regitr_overlap(itr) )
- {
- if ( itr->beg > end-1 || itr->end < start-1 ) error("query failed, incorrect region: %d-%d for %d-%d\n",itr->beg+1,itr->end+1,start,end);
- debug("\t %d-%d\n",itr->beg+1,itr->end+1);
- nhit++;
- }
- if ( nhit!=1 ) error("query failed, expected one hit, found %d: %s:%d-%d\n",nhit,chr,start,end);
-
-
- // one hit
- start = end = 10*i+1;
- if ( !regidx_overlap(idx,chr,start-1,end-1,itr) ) error("query failed, there should be a hit: %s:%d-%d\n",chr,start,end);
- debug("ok: overlap(s) found for %s:%d-%d\n",chr,start,end);
- nhit = 0;
- while ( regitr_overlap(itr) )
- {
- if ( itr->beg > end-1 || itr->end < start-1 ) error("query failed, incorrect region: %d-%d for %d-%d\n",itr->beg+1,itr->end+1,start,end);
- debug("\t %d-%d\n",itr->beg+1,itr->end+1);
- nhit++;
- }
- if ( nhit!=1 ) error("query failed, expected one hit, found %d: %s:%d-%d\n",nhit,chr,start,end);
-
-
- // two hits
- start = 10*i; end = start+1;
- if ( !regidx_overlap(idx,chr,start-1,end-1,itr) ) error("query failed, there should be a hit: %s:%d-%d\n",chr,start,end);
- debug("ok: overlap(s) found for %s:%d-%d\n",chr,start,end);
- nhit = 0;
- while ( regitr_overlap(itr) )
- {
- if ( itr->beg > end-1 || itr->end < start-1 ) error("query failed, incorrect region: %d-%d for %d-%d\n",itr->beg+1,itr->end+1,start,end);
- debug("\t %d-%d\n",itr->beg+1,itr->end+1);
- nhit++;
- }
- if ( nhit!=2 ) error("query failed, expected two hits, found %d: %s:%d-%d\n",nhit,chr,start,end);
-
- }
- regitr_destroy(itr);
- regidx_destroy(idx);
-}
-
-static void usage(void)
-{
- fprintf(bcftools_stderr, "Usage: test-regidx [OPTIONS]\n");
- fprintf(bcftools_stderr, "Options:\n");
- fprintf(bcftools_stderr, " -h, --help this help message\n");
- fprintf(bcftools_stderr, " -s, --seed <int> random seed\n");
- fprintf(bcftools_stderr, " -v, --verbose increase verbosity by giving multiple times\n");
-
- exit(1);
-}
-
-int bcftools_test-regidx_main(int argc, char **argv)
-{
- static struct option loptions[] =
- {
- {"help",0,0,'h'},
- {"verbose",0,0,'v'},
- {"seed",1,0,'s'},
- {0,0,0,0}
- };
- int c;
- int seed = (int)time(NULL);
- while ((c = getopt_long(argc, argv, "hvs:",loptions,NULL)) >= 0)
- {
- switch (c)
- {
- case 's': seed = atoi(optarg); break;
- case 'v': verbose++; break;
- default: usage(); break;
- }
- }
-
- info("Testing sequential access\n");
- test_sequential_access();
-
- info("Testing TAB\n");
- test(create_line_tab,regidx_parse_tab);
-
- info("Testing REG\n");
- test(create_line_reg,regidx_parse_reg);
-
- info("Testing BED\n");
- test(create_line_bed,regidx_parse_bed);
-
- info("Testing custom payload\n");
- test_custom_payload();
-
- int i, ntest = 1000, nreg = 50;
- srandom(seed);
- info("%d randomized tests, %d regions per test. Random seed is %d\n", ntest,nreg,seed);
- for (i=0; i<ntest; i++) test_random(nreg,1,1000);
-
- return 0;
-}
-
-
--- /dev/null
+// VariantKey
+//
+// variantkey.h
+//
+// @category Libraries
+// @author Nicola Asuni <nicola.asuni@genomicsplc.com>
+// @copyright 2017-2018 GENOMICS plc
+// @license MIT (see LICENSE)
+// @link https://github.com/genomicsplc/variantkey
+//
+// LICENSE
+//
+// Copyright (c) 2017-2018 GENOMICS plc
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+/**
+ * @file variantkey.h
+ * @brief VariantKey main functions.
+ *
+ * The functions provided here allows to generate and process a 64 bit Unsigned Integer Keys for Human Genetic Variants.
+ * The VariantKey is sortable for chromosome and position,
+ * and it is also fully reversible for variants with up to 11 bases between Reference and Alternate alleles.
+ * It can be used to sort, search and match variant-based data easily and very quickly.
+ */
+
+#ifndef VARIANTKEY_H
+#define VARIANTKEY_H
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdio.h>
+#include "hex.h"
+
+#define VKMASK_CHROM 0xF800000000000000 //!< VariantKey binary mask for CHROM [ 11111000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 ]
+#define VKMASK_POS 0x07FFFFFF80000000 //!< VariantKey binary mask for POS [ 00000111 11111111 11111111 11111111 10000000 00000000 00000000 00000000 ]
+#define VKMASK_CHROMPOS 0xFFFFFFFF80000000 //!< VariantKey binary mask for CHROM+POS [ 11111111 11111111 11111111 11111111 10000000 00000000 00000000 00000000 ]
+#define VKMASK_REFALT 0x000000007FFFFFFF //!< VariantKey binary mask for REF+ALT [ 00000000 00000000 00000000 00000000 01111111 11111111 11111111 11111111 ]
+#define VKSHIFT_CHROM 59 //!< CHROM LSB position from the VariantKey LSB
+#define VKSHIFT_POS 31 //!< POS LSB position from the VariantKey LSB
+
+/**
+ * VariantKey struct.
+ * Contains the numerically encoded VariantKey components (CHROM, POS, REF+ALT).
+ */
+typedef struct variantkey_t
+{
+ uint8_t chrom; //!< Chromosome encoded number (only the LSB 5 bit are used)
+ uint32_t pos; //!< Reference position, with the first base having position 0 (only the LSB 28 bit are used)
+ uint32_t refalt; //!< Code for Reference and Alternate allele (only the LSB 31 bits are used)
+} variantkey_t;
+
+/**
+ * Struct containing the minimum and maximum VariantKey values for range searches.
+ */
+typedef struct vkrange_t
+{
+ uint64_t min; //!< Minimum VariantKey value for any given REF+ALT encoding
+ uint64_t max; //!< Maximum VariantKey value for any given REF+ALT encoding
+} vkrange_t;
+
+/** @brief Returns chromosome numerical encoding.
+ *
+ * @param chrom Chromosome. An identifier from the reference genome, no white-space permitted.
+ * @param size Length of the chrom string, excluding the terminating null byte.
+ *
+ * @return CHROM code
+ */
+static inline uint8_t encode_chrom(const char *chrom, size_t size)
+{
+ // X > 23 ; Y > 24 ; M > 25
+ static const uint8_t onecharmap[] =
+ {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ /* M X Y */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,23,24, 0, 0, 0, 0, 0, 0,
+ /* m x y */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,23,24, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ };
+ // remove "chr" prefix
+ if ((size > 3)
+ && ((chrom[0] == 'c') || (chrom[0] == 'C'))
+ && ((chrom[1] == 'h') || (chrom[1] == 'H'))
+ && ((chrom[2] == 'r') || (chrom[2] == 'R')))
+ {
+ chrom += 3;
+ size -= 3;
+ }
+ if (size == 0)
+ {
+ return 0;
+ }
+ if ((chrom[0] <= '9') && (chrom[0] >= '0')) // Number
+ {
+ size_t i;
+ uint8_t v = (chrom[0] - '0');
+ for (i = 1; i < size; i++)
+ {
+ if ((chrom[i] > '9') || (chrom[i] < '0'))
+ {
+ return 0; // NA
+ }
+ v = ((v * 10) + (chrom[i] - '0'));
+ }
+ return v;
+ }
+ if ((size == 1) || ((size == 2) && ((chrom[1] == 'T') || (chrom[1] == 't'))))
+ {
+ return onecharmap[((uint8_t)chrom[0])];
+ }
+ return 0; // NA
+}
+
+/** @brief Decode the chromosome numerical code.
+ *
+ * @param code CHROM code.
+ * @param chrom CHROM string buffer to be returned. Its size should be enough to contain the results (max 4 bytes).
+ *
+ * @return If successful, the total number of characters written is returned,
+ * excluding the null-character appended at the end of the string,
+ * otherwise a negative number is returned in case of failure.
+ */
+static inline size_t decode_chrom(uint8_t code, char *chrom)
+{
+ if ((code < 1) || (code > 25))
+ {
+ return sprintf(chrom, "NA");
+ }
+ if (code < 23)
+ {
+ return sprintf(chrom, "%" PRIu8, code);
+ }
+ static const char *map[] = {"X", "Y", "MT"};
+ return sprintf(chrom, "%s", map[(code - 23)]);
+}
+
+static inline uint32_t encode_base(const uint8_t c)
+{
+ /*
+ Encode base:
+ A > 0
+ C > 1
+ G > 2
+ T > 3
+ */
+ static const uint32_t map[] =
+ {
+ 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+ 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+ /*A C G T*/
+ 4,0,4,1,4,4,4,2,4,4,4,4,4,4,4,4,4,4,4,4,3,4,4,4,4,4,4,4,4,4,4,4,
+ /*a c g t*/
+ 4,0,4,1,4,4,4,2,4,4,4,4,4,4,4,4,4,4,4,4,3,4,4,4,4,4,4,4,4,4,4,4,
+ 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+ 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+ 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+ 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+ };
+ return map[c];
+}
+
+static inline int encode_allele(uint32_t *h, uint8_t *bitpos, const char *str, size_t size)
+{
+ uint32_t v;
+ while (size--)
+ {
+ v = encode_base(*str++);
+ if (v > 3)
+ {
+ return -1;
+ }
+ *bitpos -= 2;
+ *h |= (v << *bitpos);
+ }
+ return 0;
+}
+
+static inline uint32_t encode_refalt_rev(const char *ref, size_t sizeref, const char *alt, size_t sizealt)
+{
+ //[******** ******** ******** ******** *RRRRAAA A1122334 45566778 8990011*]
+ uint32_t h = 0;
+ h |= ((uint32_t)(sizeref) << 27); // RRRR: length of (REF - 1)
+ h |= ((uint32_t)(sizealt) << 23); // AAAA: length of (ALT - 1)
+ uint8_t bitpos = 23;
+ if ((encode_allele(&h, &bitpos, ref, sizeref) < 0) || (encode_allele(&h, &bitpos, alt, sizealt) < 0))
+ {
+ return 0; // error code
+ }
+ return h;
+}
+
+// Mix two 32 bit hash numbers using a MurmurHash3-like algorithm
+static inline uint32_t muxhash(uint32_t k, uint32_t h)
+{
+ k *= 0xcc9e2d51;
+ k = (k >> 17) | (k << 15);
+ k *= 0x1b873593;
+ h ^= k;
+ h = (h >> 19) | (h << 13);
+ return ((h * 5) + 0xe6546b64);
+}
+
+static inline uint32_t encode_packchar(int c)
+{
+ if (c < 'A')
+ {
+ return 27;
+ }
+ if (c >= 'a')
+ {
+ return (uint32_t)(c - 'a' + 1);
+ }
+ return (uint32_t)(c - 'A' + 1);
+}
+
+// pack blocks of 6 characters in 32 bit (6 x 5 bit + 2 spare bit) [ 01111122 22233333 44444555 55666660 ]
+static inline uint32_t pack_chars_tail(const char *str, size_t size)
+{
+ uint32_t h = 0;
+ const char *pos = (str + size - 1);
+ switch (size)
+ {
+ case 5:
+ h ^= encode_packchar(*pos--) << (1 + (5 * 1));
+ // fall through
+ case 4:
+ h ^= encode_packchar(*pos--) << (1 + (5 * 2));
+ // fall through
+ case 3:
+ h ^= encode_packchar(*pos--) << (1 + (5 * 3));
+ // fall through
+ case 2:
+ h ^= encode_packchar(*pos--) << (1 + (5 * 4));
+ // fall through
+ case 1:
+ h ^= encode_packchar(*pos) << (1 + (5 * 5));
+ }
+ return h;
+}
+
+static inline uint32_t pack_chars(const char *str)
+{
+ const char *pos = (str + 5);
+ return ((encode_packchar(*pos) << 1)
+ ^ (encode_packchar(*(pos-1)) << (1 + (5 * 1)))
+ ^ (encode_packchar(*(pos-2)) << (1 + (5 * 2)))
+ ^ (encode_packchar(*(pos-3)) << (1 + (5 * 3)))
+ ^ (encode_packchar(*(pos-4)) << (1 + (5 * 4)))
+ ^ (encode_packchar(*(pos-5)) << (1 + (5 * 5))));
+}
+
+// Return a 32 bit hash of a nucleotide string
+static inline uint32_t hash32(const char *str, size_t size)
+{
+ uint32_t h = 0;
+ size_t len = 6;
+ while (size >= len)
+ {
+ h = muxhash(pack_chars(str), h);
+ str += len;
+ size -= len;
+ }
+ if (size > 0)
+ {
+ h = muxhash(pack_chars_tail(str, size), h);
+ }
+ return h;
+}
+
+static inline uint32_t encode_refalt_hash(const char *ref, size_t sizeref, const char *alt, size_t sizealt)
+{
+ // 0x3 is the separator character between REF and ALT [00000000 00000000 00000000 00000011]
+ uint32_t h = muxhash(hash32(alt, sizealt), muxhash(0x3, hash32(ref, sizeref)));
+ // MurmurHash3 finalization mix - force all bits of a hash block to avalanche
+ h ^= h >> 16;
+ h *= 0x85ebca6b;
+ h ^= h >> 13;
+ h *= 0xc2b2ae35;
+ h ^= h >> 16;
+ return ((h >> 1) | 0x1); // 0x1 is the set bit to indicate HASH mode [00000000 00000000 00000000 00000001]
+}
+
+/** @brief Returns reference+alternate numerical encoding.
+ *
+ * @param ref Reference allele. String containing a sequence of nucleotide letters.
+ * The value in the pos field refers to the position of the first nucleotide in the String.
+ * Characters must be A-Z, a-z or *
+ * @param sizeref Length of the ref string, excluding the terminating null byte.
+ * @param alt Alternate non-reference allele string.
+ * Characters must be A-Z, a-z or *
+ * @param sizealt Length of the alt string, excluding the terminating null byte.
+ *
+ * @return REF+ALT code
+ */
+static inline uint32_t encode_refalt(const char *ref, size_t sizeref, const char *alt, size_t sizealt)
+{
+ if ((sizeref + sizealt) <= 11)
+ {
+ uint32_t h = encode_refalt_rev(ref, sizeref, alt, sizealt);
+ if (h != 0)
+ {
+ return h;
+ }
+ }
+ return encode_refalt_hash(ref, sizeref, alt, sizealt);
+}
+
+static inline char decode_base(uint32_t code, int bitpos)
+{
+ static const char base[4] = {'A', 'C', 'G', 'T'};
+ return base[((code >> bitpos) & 0x3)]; // 0x3 is the 2 bit mask [00000011]
+}
+
+static inline size_t decode_refalt_rev(uint32_t code, char *ref, size_t *sizeref, char *alt, size_t *sizealt)
+{
+ *sizeref = (size_t)((code & 0x78000000) >> 27); // [01111000 00000000 00000000 00000000]
+ *sizealt = (size_t)((code & 0x07800000) >> 23); // [00000111 10000000 00000000 00000000]
+ switch (*sizeref)
+ {
+ case 10:
+ ref[9] = decode_base(code, (3 + (2 * 0)));
+ // fall through
+ case 9:
+ ref[8] = decode_base(code, (3 + (2 * 1)));
+ // fall through
+ case 8:
+ ref[7] = decode_base(code, (3 + (2 * 2)));
+ // fall through
+ case 7:
+ ref[6] = decode_base(code, (3 + (2 * 3)));
+ // fall through
+ case 6:
+ ref[5] = decode_base(code, (3 + (2 * 4)));
+ // fall through
+ case 5:
+ ref[4] = decode_base(code, (3 + (2 * 5)));
+ // fall through
+ case 4:
+ ref[3] = decode_base(code, (3 + (2 * 6)));
+ // fall through
+ case 3:
+ ref[2] = decode_base(code, (3 + (2 * 7)));
+ // fall through
+ case 2:
+ ref[1] = decode_base(code, (3 + (2 * 8)));
+ // fall through
+ case 1:
+ ref[0] = decode_base(code, (3 + (2 * 9)));
+ }
+ ref[*sizeref] = 0;
+ uint8_t bitpos = (23 - ((*sizeref) << 1));
+ switch (*sizealt)
+ {
+ case 10:
+ alt[9] = decode_base(code, bitpos - (2 * 10));
+ // fall through
+ case 9:
+ alt[8] = decode_base(code, bitpos - (2 * 9));
+ // fall through
+ case 8:
+ alt[7] = decode_base(code, bitpos - (2 * 8));
+ // fall through
+ case 7:
+ alt[6] = decode_base(code, bitpos - (2 * 7));
+ // fall through
+ case 6:
+ alt[5] = decode_base(code, bitpos - (2 * 6));
+ // fall through
+ case 5:
+ alt[4] = decode_base(code, bitpos - (2 * 5));
+ // fall through
+ case 4:
+ alt[3] = decode_base(code, bitpos - (2 * 4));
+ // fall through
+ case 3:
+ alt[2] = decode_base(code, bitpos - (2 * 3));
+ // fall through
+ case 2:
+ alt[1] = decode_base(code, bitpos - (2 * 2));
+ // fall through
+ case 1:
+ alt[0] = decode_base(code, bitpos - (2 * 1));
+ }
+ alt[*sizealt] = 0;
+ return (*sizeref + *sizealt);
+}
+
+/** @brief Decode the 32 bit REF+ALT code if reversible (if it has 11 or less bases in total and only contains ACGT letters).
+ *
+ * @param code REF+ALT code
+ * @param ref REF string buffer to be returned.
+ * @param sizeref Pointer to the size of the ref buffer, excluding the terminating null byte.
+ * This will contain the final ref size.
+ * @param alt ALT string buffer to be returned.
+ * @param sizealt Pointer to the size of the alt buffer, excluding the terminating null byte.
+ * This will contain the final alt size.
+ *
+ * @return If the code is reversible, then the total number of characters of REF+ALT is returned.
+ * Otherwise 0 is returned.
+ */
+static inline size_t decode_refalt(uint32_t code, char *ref, size_t *sizeref, char *alt, size_t *sizealt)
+{
+ if (code & 0x1) // check last bit
+ {
+ return 0; // non-reversible encoding
+ }
+ return decode_refalt_rev(code, ref, sizeref, alt, sizealt);
+}
+
+/** @brief Returns a 64 bit variant key based on the pre-encoded CHROM, POS (0-based) and REF+ALT.
+ *
+ * @param chrom Encoded Chromosome (see encode_chrom).
+ * @param pos Position. The reference position, with the first base having position 0.
+ * @param refalt Encoded Reference + Alternate (see encode_refalt).
+ *
+ * @return VariantKey 64 bit code.
+ */
+static inline uint64_t encode_variantkey(uint8_t chrom, uint32_t pos, uint32_t refalt)
+{
+ return (((uint64_t)chrom << VKSHIFT_CHROM) | ((uint64_t)pos << VKSHIFT_POS) | (uint64_t)refalt);
+}
+
+/** @brief Extract the CHROM code from VariantKey.
+ *
+ * @param vk VariantKey code.
+ *
+ * @return CHROM code.
+ */
+static inline uint8_t extract_variantkey_chrom(uint64_t vk)
+{
+ return (uint8_t)((vk & VKMASK_CHROM) >> VKSHIFT_CHROM);
+}
+
+/** @brief Extract the POS code from VariantKey.
+ *
+ * @param vk VariantKey code.
+ *
+ * @return POS.
+ */
+static inline uint32_t extract_variantkey_pos(uint64_t vk)
+{
+ return (uint32_t)((vk & VKMASK_POS) >> VKSHIFT_POS);
+}
+
+/** @brief Extract the REF+ALT code from VariantKey.
+ *
+ * @param vk VariantKey code.
+ *
+ * @return REF+ALT code.
+ */
+static inline uint32_t extract_variantkey_refalt(uint64_t vk)
+{
+ return (uint32_t)(vk & VKMASK_REFALT);
+}
+
+/** @brief Decode a VariantKey code and returns the components as variantkey_t structure.
+ *
+ * @param code VariantKey code.
+ * @param vk Decoded variantkey structure.
+ */
+static inline void decode_variantkey(uint64_t code, variantkey_t *vk)
+{
+ vk->chrom = extract_variantkey_chrom(code);
+ vk->pos = extract_variantkey_pos(code);
+ vk->refalt = extract_variantkey_refalt(code);
+}
+
+/** @brief Returns a 64 bit variant key based on CHROM, POS (0-based), REF, ALT.
+ *
+ * @param chrom Chromosome. An identifier from the reference genome, no white-space or leading zeros permitted.
+ * @param sizechrom Length of the chrom string, excluding the terminating null byte.
+ * @param pos Position. The reference position, with the first base having position 0.
+ * @param ref Reference allele. String containing a sequence of nucleotide letters.
+ * The value in the pos field refers to the position of the first nucleotide in the String.
+ * Characters must be A-Z, a-z or *
+ * @param sizeref Length of the ref string, excluding the terminating null byte.
+ * @param alt Alternate non-reference allele string.
+ * Characters must be A-Z, a-z or *
+ * @param sizealt Length of the alt string, excluding the terminating null byte.
+ *
+ * @return VariantKey 64 bit code.
+ */
+static inline uint64_t variantkey(const char *chrom, size_t sizechrom, uint32_t pos, const char *ref, size_t sizeref, const char *alt, size_t sizealt)
+{
+ return encode_variantkey(encode_chrom(chrom, sizechrom), pos, encode_refalt(ref, sizeref, alt, sizealt));
+}
+
+/** @brief Returns minimum and maximum VariantKeys for range searches.
+ *
+ * @param chrom Chromosome encoded number.
+ * @param pos_min Start reference position, with the first base having position 0.
+ * @param pos_max End reference position, with the first base having position 0.
+ * @param range VariantKey range values.
+ */
+static inline void variantkey_range(uint8_t chrom, uint32_t pos_min, uint32_t pos_max, vkrange_t *range)
+{
+ uint64_t c = ((uint64_t)chrom << VKSHIFT_CHROM);
+ range->min = (c | ((uint64_t)pos_min << VKSHIFT_POS));
+ range->max = (c | ((uint64_t)pos_max << VKSHIFT_POS) | VKMASK_REFALT);
+}
+
+static inline int8_t compare_uint64_t(uint64_t a, uint64_t b)
+{
+ return (a < b) ? -1 : (a > b);
+}
+
+/** @brief Compares two VariantKeys by chromosome only.
+ *
+ * @param vka The first VariantKey to be compared.
+ * @param vkb The second VariantKey to be compared.
+ *
+ * @return -1 if the first chromosome is smaller than the second, 0 if they are equal and 1 if the first is greater than the second.
+ */
+static inline int8_t compare_variantkey_chrom(uint64_t vka, uint64_t vkb)
+{
+ return compare_uint64_t((vka >> VKSHIFT_CHROM), (vkb >> VKSHIFT_CHROM));
+}
+
+/** @brief Compares two VariantKeys by chromosome and position.
+ *
+ * @param vka The first VariantKey to be compared.
+ * @param vkb The second VariantKey to be compared.
+ *
+ * @return -1 if the first CHROM+POS is smaller than the second, 0 if they are equal and 1 if the first is greater than the second.
+ */
+static inline int8_t compare_variantkey_chrom_pos(uint64_t vka, uint64_t vkb)
+{
+ return compare_uint64_t((vka >> VKSHIFT_POS), (vkb >> VKSHIFT_POS));
+}
+
+/** @brief Returns VariantKey hexadecimal string (16 characters).
+ *
+ * The string represent a 64 bit number or:
+ * - 5 bit for CHROM
+ * - 28 bit for POS
+ * - 31 bit for REF+ALT
+ *
+ * @param vk VariantKey code.
+ * @param str String buffer to be returned (it must be sized 17 bytes at least).
+ *
+ * @return Upon successful return, these function returns the number of characters processed
+ * (excluding the null byte used to end output to strings).
+ * If the buffer size is not sufficient, then the return value is the number of characters required for
+ * buffer string, including the terminating null byte.
+ */
+static inline size_t variantkey_hex(uint64_t vk, char *str)
+{
+ return hex_uint64_t(vk, str);
+}
+
+/** @brief Parses a VariantKey hexadecimal string and returns the code.
+ *
+ * @param vs VariantKey hexadecimal string (it must contain 16 hexadecimal characters).
+ *
+ * @return A VariantKey code.
+ */
+static inline uint64_t parse_variantkey_hex(const char *vs)
+{
+ return parse_hex_uint64_t(vs);
+}
+
+#endif // VARIANTKEY_H
/* vcfannotate.c -- Annotate and edit VCF/BCF files.
- Copyright (C) 2013-2018 Genome Research Ltd.
+ Copyright (C) 2013-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#include <sys/types.h>
#include <dirent.h>
#include <math.h>
+#include <inttypes.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/kseq.h>
#include <htslib/khash_str2int.h>
-#include <dlfcn.h>
#include "bcftools.h"
#include "vcmp.h"
#include "filter.h"
#include "convert.h"
#include "smpl_ilist.h"
+#include "regidx.h"
struct _args_t;
}
annot_line_t;
-#define REPLACE_MISSING 0 // replace only missing values
-#define REPLACE_ALL 1 // replace both missing and existing values
-#define REPLACE_NON_MISSING 2 // replace only if tgt is not missing
-#define SET_OR_APPEND 3 // set new value if missing or non-existent, append otherwise
+#define REPLACE_MISSING 0 // replace only missing values
+#define REPLACE_ALL 1 // replace both missing and existing values
+#define REPLACE_NON_MISSING 2 // replace only if tgt is not missing
+#define SET_OR_APPEND 3 // set new value if missing or non-existent, append otherwise
+#define MM_FIRST 0 // if multiple annotation lines overlap a VCF record, use the first, discarding the rest
+#define MM_APPEND 1 // append, possibly multiple times
+#define MM_UNIQUE 2 // append, only unique values
+#define MM_SUM 3
+#define MM_AVG 4
+#define MM_MIN 5
+#define MM_MAX 6
typedef struct _annot_col_t
{
int icol, replace, number; // number: one of BCF_VL_* types
char *hdr_key_src, *hdr_key_dst;
int (*setter)(struct _args_t *, bcf1_t *, struct _annot_col_t *, void*);
+ int merge_method; // one of the MM_* defines
+ khash_t(str2int) *mm_str_hash; // lookup table to ensure uniqueness of added string values
+ kstring_t mm_kstr;
+ double
+ mm_dbl_nalloc, // the allocated size --merge-logic values array
+ mm_dbl_nused, // the number of used elements in the mm_dbl array
+ mm_dbl_ndat, // the number of merged rows (for calculating the average)
+ *mm_dbl;
}
annot_col_t;
int output_type, n_threads;
bcf_sr_regions_t *tgts;
+ regidx_t *tgt_idx;
+ regitr_t *tgt_itr;
+ int tgt_is_bed;
+
filter_t *filter;
char *filter_str;
int filter_logic; // include or exclude sites which match the filters? One of FLT_INCLUDE/FLT_EXCLUDE
vcmp_t *vcmp; // for matching annotation and VCF lines by allele
annot_line_t *alines; // buffered annotation lines
int nalines, malines;
- int ref_idx, alt_idx, chr_idx, from_idx, to_idx; // -1 if not present
+ int ref_idx, alt_idx, chr_idx, beg_idx, end_idx; // -1 if not present
annot_col_t *cols; // column indexes and setters
int ncols;
char **argv, *output_fname, *targets_fname, *regions_list, *header_fname;
char *remove_annots, *columns, *rename_chrs, *sample_names, *mark_sites;
- int argc, drop_header, record_cmd_line, tgts_is_vcf, mark_sites_logic;
+ char *merge_method_str;
+ int argc, drop_header, record_cmd_line, tgts_is_vcf, mark_sites_logic, force, single_overlaps;
}
args_t;
char *msprintf(const char *fmt, ...);
+int parse_with_payload(const char *line, char **chr_beg, char **chr_end, uint32_t *beg, uint32_t *end, void *payload, void *usr)
+{
+ args_t *args = (args_t*) usr;
+ int ret = args->tgt_is_bed ? regidx_parse_bed(line, chr_beg, chr_end, beg, end, NULL, NULL) : regidx_parse_tab(line, chr_beg, chr_end, beg, end, NULL, NULL);
+ if ( ret<0 ) return ret;
+ *((char **)payload) = strdup(line);
+ return 0;
+}
+void free_payload(void *payload)
+{
+ char *str = *((char**)payload);
+ free(str);
+}
+
void remove_id(args_t *args, bcf1_t *line, rm_tag_t *tag)
{
bcf_update_id(args->hdr,line,NULL);
}
void remove_filter(args_t *args, bcf1_t *line, rm_tag_t *tag)
{
+ if ( tag->key && tag->hdr_id<0 )
+ {
+ error("Error: Cannot proceed, not even with the --force option, bad things could happen.\n"
+ " Note that \"bcftools annotate -x FILTER\" can be used to remove ALL filters.\n"
+ " Even better, use \"bcftools view -h\" and \"bcftools reheader\" to fix the header!\n"
+ );
+ }
if ( !tag->key ) bcf_update_filter(args->hdr, line, NULL, args->flt_keep_pass);
else bcf_remove_filter(args->hdr, line, tag->hdr_id, args->flt_keep_pass);
}
memmove(&hdr->hrec[i],&hdr->hrec[i+1],(hdr->nhrec-i)*sizeof(bcf_hrec_t*));
bcf_hrec_destroy(hrec);
}
- if ( nrm ) bcf_hdr_sync(hdr);
+ if ( nrm ) {
+ if (bcf_hdr_sync(hdr) < 0)
+ error_errno("[%s] Failed to update header", __func__);
+ }
}
static void init_remove_annots(args_t *args)
tag->handler = remove_filter;
tag->key = strdup(str.s);
tag->hdr_id = bcf_hdr_id2int(args->hdr, BCF_DT_ID, tag->key);
- if ( !bcf_hdr_idinfo_exists(args->hdr,BCF_HL_FLT,tag->hdr_id) ) error("Cannot remove %s, not defined in the header.\n", str.s);
- if ( !args->keep_sites ) bcf_hdr_remove(args->hdr_out,BCF_HL_FLT,tag->key);
+ if ( !bcf_hdr_idinfo_exists(args->hdr,BCF_HL_FLT,tag->hdr_id) )
+ {
+ if ( args->keep_sites )
+ error("Error: The filter \"%s\" is not defined in the header, cannot use the -k option\n", str.s);
+ else
+ fprintf(stderr,"Warning: The filter \"%s\" is not defined in the header\n", str.s);
+ }
+ else if ( !args->keep_sites ) bcf_hdr_remove(args->hdr_out,BCF_HL_FLT,tag->key);
}
else
{
int id = bcf_hdr_id2int(args->hdr,BCF_DT_ID,str.s);
if ( !bcf_hdr_idinfo_exists(args->hdr,type,id) )
{
- fprintf(stderr,"Warning: The tag \"%s\" not defined in the header\n", str.s);
- args->nrm--;
+ if ( args->keep_sites )
+ error("Error: The tag \"%s\" is not defined in the header, cannot use the -k option\n", str.s);
+ else
+ fprintf(stderr,"Warning: The tag \"%s\" not defined in the header\n", str.s);
+
+ tag->key = strdup(str.s);
+ if ( type==BCF_HL_INFO ) tag->handler = remove_info_tag;
+ else if ( type==BCF_HL_FMT ) tag->handler = remove_format_tag;
}
else if ( (type==BCF_HL_FMT && keep_fmt) || (type==BCF_HL_INFO && keep_info) )
{
}
khash_str2int_destroy_free(keep);
if ( !args->nrm ) error("No matching tag in -x %s\n", args->remove_annots);
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
}
static void init_header_lines(args_t *args)
{
if ( bcf_hdr_append(args->hdr_out,str.s) ) error("Could not parse %s: %s\n", args->header_fname, str.s);
bcf_hdr_append(args->hdr,str.s); // the input file may not have the header line if run with -h (and nothing else)
}
- hts_close(file);
+ if ( hts_close(file)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->header_fname);
free(str.s);
- bcf_hdr_sync(args->hdr_out);
- bcf_hdr_sync(args->hdr);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update output header", __func__);
+ if (bcf_hdr_sync(args->hdr) < 0)
+ error_errno("[%s] Failed to update input header", __func__);
}
static int setter_filter(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with FILTER (yet?)\n");
+
// note: so far this works only with one filter, not a list of filters
annot_line_t *tab = (annot_line_t*) data;
if ( tab->cols[col->icol] && tab->cols[col->icol][0]=='.' && !tab->cols[col->icol][1] ) return 0; // don't replace with "."
}
static int setter_id(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with ID (yet?)\n");
+
// possible cases:
// IN ANNOT OUT ACHIEVED_BY
// x y x -c +ID
}
static int setter_qual(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with QUAL (yet?)\n");
+
annot_line_t *tab = (annot_line_t*) data;
char *str = tab->cols[col->icol];
if ( str[0]=='.' && str[1]==0 ) return 0; // empty
line->qual = strtod(str, &str);
if ( str == tab->cols[col->icol] )
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
return 0;
}
static int vcf_setter_qual(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
}
static int setter_info_flag(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with INFO type=Flag (yet?)\n");
+
annot_line_t *tab = (annot_line_t*) data;
char *str = tab->cols[col->icol];
if ( str[0]=='.' && str[1]==0 ) return 0;
if ( str[0]=='1' && str[1]==0 ) return bcf_update_info_flag(args->hdr_out,line,col->hdr_key_dst,NULL,1);
if ( str[0]=='0' && str[1]==0 ) return bcf_update_info_flag(args->hdr_out,line,col->hdr_key_dst,NULL,0);
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
return -1;
}
static int vcf_setter_info_flag(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
static int setter_ARinfo_int32(args_t *args, bcf1_t *line, annot_col_t *col, int nals, char **als, int ntmpi)
{
if ( col->number==BCF_VL_A && ntmpi!=nals-1 && (ntmpi!=1 || args->tmpi[0]!=bcf_int32_missing || args->tmpi[1]!=bcf_int32_vector_end) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", ntmpi,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", ntmpi,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
else if ( col->number==BCF_VL_R && ntmpi!=nals && (ntmpi!=1 || args->tmpi[0]!=bcf_int32_missing || args->tmpi[1]!=bcf_int32_vector_end) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", ntmpi,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", ntmpi,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ndst = col->number==BCF_VL_A ? line->n_allele - 1 : line->n_allele;
int *map = vcmp_map_ARvalues(args->vcmp,ndst,nals,als,line->n_allele,line->d.allele);
- if ( !map ) error("REF alleles not compatible at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !map ) error("REF alleles not compatible at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
// fill in any missing values in the target VCF (or all, if not present)
int ntmpi2 = bcf_get_info_float(args->hdr, line, col->hdr_key_dst, &args->tmpi2, &args->mtmpi2);
static int setter_info_int(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
annot_line_t *tab = (annot_line_t*) data;
- char *str = tab->cols[col->icol], *end = str;
- if ( str[0]=='.' && str[1]==0 ) return 0;
- int ntmpi = 0;
- while ( *end )
+ if ( !tab )
+ {
+ if ( col->merge_method!=MM_SUM && col->merge_method!=MM_AVG && col->merge_method!=MM_MIN && col->merge_method!=MM_MAX && col->merge_method!=MM_APPEND )
+ error("Error: at the moment only the sum,avg,min,max,append options are supported with --merge-logic for INFO type=Integer\n");
+ }
+
+ int i,ntmpi = 0;
+ if ( tab )
{
- int val = strtol(str, &end, 10);
- if ( end==str )
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
- ntmpi++;
+ char *str = tab->cols[col->icol], *end = str;
+ if ( str[0]=='.' && str[1]==0 ) return 0;
+
+ while ( *end )
+ {
+ int val = strtol(str, &end, 10);
+ if ( end==str )
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
+ ntmpi++;
+ hts_expand(int32_t,ntmpi,args->mtmpi,args->tmpi);
+ args->tmpi[ntmpi-1] = val;
+ str = end+1;
+ }
+ if ( col->merge_method!=MM_FIRST )
+ {
+ if ( !col->mm_dbl_nused )
+ {
+ col->mm_dbl_nused = ntmpi;
+ hts_expand(double,col->mm_dbl_nused,col->mm_dbl_nalloc,col->mm_dbl);
+ for (i=0; i<ntmpi; i++)
+ col->mm_dbl[i] = args->tmpi[i];
+ }
+ else
+ {
+ if ( col->merge_method==MM_APPEND )
+ {
+ int nori = col->mm_dbl_nused;
+ col->mm_dbl_nused += ntmpi;
+ hts_expand(double,col->mm_dbl_nused,col->mm_dbl_nalloc,col->mm_dbl);
+ for (i=0; i<ntmpi; i++)
+ col->mm_dbl[i+nori] = args->tmpi[i];
+ }
+ else
+ {
+ if ( ntmpi!=col->mm_dbl_nused ) error("Error: cannot merge fields of unequal length\n");
+ if ( col->merge_method==MM_SUM || col->merge_method==MM_AVG )
+ for (i=0; i<ntmpi; i++) col->mm_dbl[i] += args->tmpi[i];
+ else if ( col->merge_method==MM_MIN )
+ for (i=0; i<ntmpi; i++) { if ( col->mm_dbl[i] > args->tmpi[i] ) col->mm_dbl[i] = args->tmpi[i]; }
+ else if ( col->merge_method==MM_MAX )
+ for (i=0; i<ntmpi; i++) { if ( col->mm_dbl[i] < args->tmpi[i] ) col->mm_dbl[i] = args->tmpi[i]; }
+ }
+ }
+ col->mm_dbl_ndat++;
+ }
+ }
+ else if ( col->merge_method==MM_SUM || col->merge_method==MM_MIN || col->merge_method==MM_MAX || col->merge_method==MM_APPEND )
+ {
+ ntmpi = col->mm_dbl_nused;
hts_expand(int32_t,ntmpi,args->mtmpi,args->tmpi);
- args->tmpi[ntmpi-1] = val;
- str = end+1;
+ for (i=0; i<ntmpi; i++) args->tmpi[i] = col->mm_dbl[i];
+ col->mm_dbl_nused = col->mm_dbl_ndat = 0;
+ }
+ else if ( col->merge_method==MM_AVG )
+ {
+ ntmpi = col->mm_dbl_nused;
+ hts_expand(int32_t,ntmpi,args->mtmpi,args->tmpi);
+ for (i=0; i<ntmpi; i++) args->tmpi[i] = col->mm_dbl[i]/col->mm_dbl_ndat;
+ col->mm_dbl_nused = col->mm_dbl_ndat = 0;
}
if ( col->number==BCF_VL_A || col->number==BCF_VL_R )
static int setter_ARinfo_real(args_t *args, bcf1_t *line, annot_col_t *col, int nals, char **als, int ntmpf)
{
if ( col->number==BCF_VL_A && ntmpf!=nals-1 && (ntmpf!=1 || !bcf_float_is_missing(args->tmpf[0]) || !bcf_float_is_vector_end(args->tmpf[0])) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", ntmpf,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", ntmpf,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
else if ( col->number==BCF_VL_R && ntmpf!=nals && (ntmpf!=1 || !bcf_float_is_missing(args->tmpf[0]) || !bcf_float_is_vector_end(args->tmpf[0])) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", ntmpf,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", ntmpf,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ndst = col->number==BCF_VL_A ? line->n_allele - 1 : line->n_allele;
int *map = vcmp_map_ARvalues(args->vcmp,ndst,nals,als,line->n_allele,line->d.allele);
- if ( !map ) error("REF alleles not compatible at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !map ) error("REF alleles not compatible at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
// fill in any missing values in the target VCF (or all, if not present)
int ntmpf2 = bcf_get_info_float(args->hdr, line, col->hdr_key_dst, &args->tmpf2, &args->mtmpf2);
static int setter_info_real(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
annot_line_t *tab = (annot_line_t*) data;
- char *str = tab->cols[col->icol], *end = str;
- if ( str[0]=='.' && str[1]==0 ) return 0;
- int ntmpf = 0;
- while ( *end )
+ if ( !tab )
+ {
+ if ( col->merge_method!=MM_SUM && col->merge_method!=MM_AVG && col->merge_method!=MM_MIN && col->merge_method!=MM_MAX && col->merge_method!=MM_APPEND )
+ error("Error: at the moment only the sum,avg,min,max,append options are supported with --merge-logic for INFO type=Float\n");
+ }
+
+ int i,ntmpf = 0;
+ if ( tab )
+ {
+ char *str = tab->cols[col->icol], *end = str;
+ if ( str[0]=='.' && str[1]==0 ) return 0;
+
+ while ( *end )
+ {
+ double val = strtod(str, &end);
+ if ( end==str )
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
+ ntmpf++;
+ hts_expand(float,ntmpf,args->mtmpf,args->tmpf);
+ args->tmpf[ntmpf-1] = val;
+ str = end+1;
+ }
+ if ( col->merge_method!=MM_FIRST )
+ {
+ if ( !col->mm_dbl_nused )
+ {
+ col->mm_dbl_nused = ntmpf;
+ hts_expand(double,col->mm_dbl_nused,col->mm_dbl_nalloc,col->mm_dbl);
+ for (i=0; i<ntmpf; i++)
+ col->mm_dbl[i] = args->tmpf[i];
+ }
+ else
+ {
+ if ( col->merge_method==MM_APPEND )
+ {
+ int nori = col->mm_dbl_nused;
+ col->mm_dbl_nused += ntmpf;
+ hts_expand(double,col->mm_dbl_nused,col->mm_dbl_nalloc,col->mm_dbl);
+ for (i=0; i<ntmpf; i++)
+ col->mm_dbl[i+nori] = args->tmpf[i];
+ }
+ else
+ {
+ if ( ntmpf!=col->mm_dbl_nused ) error("Error: cannot merge fields of unequal length\n");
+ if ( col->merge_method==MM_SUM || col->merge_method==MM_AVG )
+ for (i=0; i<ntmpf; i++) col->mm_dbl[i] += args->tmpf[i];
+ else if ( col->merge_method==MM_MIN )
+ for (i=0; i<ntmpf; i++) { if ( col->mm_dbl[i] > args->tmpf[i] ) col->mm_dbl[i] = args->tmpf[i]; }
+ else if ( col->merge_method==MM_MAX )
+ for (i=0; i<ntmpf; i++) { if ( col->mm_dbl[i] < args->tmpf[i] ) col->mm_dbl[i] = args->tmpf[i]; }
+ }
+ }
+ col->mm_dbl_ndat++;
+ }
+ }
+ else if ( col->merge_method==MM_SUM || col->merge_method==MM_MIN || col->merge_method==MM_MAX || col->merge_method==MM_APPEND )
{
- double val = strtod(str, &end);
- if ( end==str )
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
- ntmpf++;
- hts_expand(float,ntmpf,args->mtmpf,args->tmpf);
- args->tmpf[ntmpf-1] = val;
- str = end+1;
+ ntmpf = col->mm_dbl_nused;
+ hts_expand(int32_t,ntmpf,args->mtmpf,args->tmpf);
+ for (i=0; i<ntmpf; i++) args->tmpf[i] = col->mm_dbl[i];
+ col->mm_dbl_nused = col->mm_dbl_ndat = 0;
+ }
+ else if ( col->merge_method==MM_AVG )
+ {
+ ntmpf = col->mm_dbl_nused;
+ hts_expand(int32_t,ntmpf,args->mtmpf,args->tmpf);
+ for (i=0; i<ntmpf; i++) args->tmpf[i] = col->mm_dbl[i]/col->mm_dbl_ndat;
+ col->mm_dbl_nused = col->mm_dbl_ndat = 0;
}
if ( col->number==BCF_VL_A || col->number==BCF_VL_R )
int copy_string_field(char *src, int isrc, int src_len, kstring_t *dst, int idst); // see vcfmerge.c
static int setter_ARinfo_string(args_t *args, bcf1_t *line, annot_col_t *col, int nals, char **als)
{
+ assert( col->merge_method==MM_FIRST );
+
int nsrc = 1, lsrc = 0;
while ( args->tmps[lsrc] )
{
lsrc++;
}
if ( col->number==BCF_VL_A && nsrc!=nals-1 && (nsrc!=1 || args->tmps[0]!='.' || args->tmps[1]!=0 ) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", nsrc,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", nsrc,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
else if ( col->number==BCF_VL_R && nsrc!=nals && (nsrc!=1 || args->tmps[0]!='.' || args->tmps[1]!=0 ) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", nsrc,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", nsrc,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ndst = col->number==BCF_VL_A ? line->n_allele - 1 : line->n_allele;
int *map = vcmp_map_ARvalues(args->vcmp,ndst,nals,als,line->n_allele,line->d.allele);
- if ( !map ) error("REF alleles not compatible at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !map ) error("REF alleles not compatible at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
// fill in any missing values in the target VCF (or all, if not present)
int i, empty = 0, nstr, mstr = args->tmpks.m;
bcf_update_info_string(args->hdr_out,line,col->hdr_key_dst,args->tmpks.s);
return 0;
}
+void khash_str2int_clear_free(void *_hash)
+{
+ khash_t(str2int) *hash = (khash_t(str2int)*)_hash;
+ khint_t k;
+ if (hash == 0) return;
+ for (k = 0; k < kh_end(hash); ++k)
+ if (kh_exist(hash, k)) free((char*)kh_key(hash, k));
+ kh_clear(str2int, hash);
+}
static int setter_info_str(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( col->replace==REPLACE_MISSING && col->number!=BCF_VL_A && col->number!=BCF_VL_R )
+ {
+ int ret = bcf_get_info_string(args->hdr, line, col->hdr_key_dst, &args->tmps2, &args->mtmps2);
+ if ( ret>0 && (args->tmps2[0]!='.' || args->tmps2[1]!=0) ) return 0;
+ }
+
annot_line_t *tab = (annot_line_t*) data;
- int len = strlen(tab->cols[col->icol]);
- if ( !len ) return 0;
- hts_expand(char,len+1,args->mtmps,args->tmps);
- memcpy(args->tmps,tab->cols[col->icol],len+1);
- if ( args->tmps[0]=='.' && args->tmps[1]==0 ) return 0;
+
+ int len = 0;
+ if ( tab )
+ {
+ len = strlen(tab->cols[col->icol]);
+ if ( !len ) return 0;
+ if ( len==1 && tab->cols[col->icol][0]=='.' ) return 0;
+ }
- if ( col->number==BCF_VL_A || col->number==BCF_VL_R )
- return setter_ARinfo_string(args,line,col,tab->nals,tab->als);
+ if ( col->merge_method!=MM_FIRST )
+ {
+ if ( col->number==BCF_VL_A || col->number==BCF_VL_R )
+ error("Error: the --merge-logic option cannot be used with INFO tags Type=String,Number={A,R,G}\n");
- if ( col->replace==REPLACE_MISSING )
+ if ( data )
+ {
+ assert( col->merge_method==MM_APPEND || col->merge_method==MM_UNIQUE );
+ if ( col->merge_method==MM_UNIQUE )
+ {
+ if ( !col->mm_str_hash ) col->mm_str_hash = (khash_t(str2int)*)khash_str2int_init();
+ if ( khash_str2int_has_key(col->mm_str_hash, tab->cols[col->icol]) ) return 0;
+ khash_str2int_inc(col->mm_str_hash, strdup(tab->cols[col->icol]));
+ }
+
+ if ( col->mm_kstr.l ) kputc(',',&col->mm_kstr);
+ kputs(tab->cols[col->icol], &col->mm_kstr);
+ return 0;
+ }
+
+ if ( col->mm_kstr.l )
+ {
+ hts_expand(char,col->mm_kstr.l+1,args->mtmps,args->tmps);
+ memcpy(args->tmps,col->mm_kstr.s,col->mm_kstr.l+1);
+ }
+ else
+ return 0;
+
+ if ( !data ) // flush the line
+ {
+ if ( col->merge_method==MM_UNIQUE )
+ khash_str2int_clear_free(col->mm_str_hash);
+ col->mm_kstr.l = 0;
+ }
+ }
+ else
{
- int ret = bcf_get_info_string(args->hdr, line, col->hdr_key_dst, &args->tmps2, &args->mtmps2);
- if ( ret>0 && (args->tmps2[0]!='.' || args->tmps2[1]!=0) ) return 0;
+ assert(tab);
+ hts_expand(char,len+1,args->mtmps,args->tmps);
+ memcpy(args->tmps,tab->cols[col->icol],len+1);
+
+ if ( col->number==BCF_VL_A || col->number==BCF_VL_R )
+ return setter_ARinfo_string(args,line,col,tab->nals,tab->als);
}
bcf_update_info_string(args->hdr_out,line,col->hdr_key_dst,args->tmps);
bcf_update_info_string(args->hdr_out,line,col->hdr_key_dst,args->tmps);
return 0;
}
+static int genotypes_to_string(args_t *args, int nsrc1, int32_t *src, int nsmpl_dst, kstring_t *str)
+{
+ int i, isrc, idst;
+ int blen = nsrc1 > 1 ? nsrc1 + 1 : 1; // typically the genotypes take three bytes 0/1, no 0-termination is needed
+
+gt_length_too_big:
+ str->l = 0;
+ for (idst=0; idst<nsmpl_dst; idst++)
+ {
+ isrc = args->sample_map ? args->sample_map[idst] : idst;
+ if ( isrc==-1 )
+ {
+ kputc_('.', str);
+ for (i=1; i < blen; i++) kputc_(0, str);
+ continue;
+ }
+
+ size_t plen = str->l;
+ int32_t *ptr = src + isrc*nsrc1;
+ for (i=0; i<nsrc1 && ptr[i]!=bcf_int32_vector_end; i++)
+ {
+ if ( i ) kputc("/|"[bcf_gt_is_phased(ptr[i])], str);
+ if ( bcf_gt_is_missing(ptr[i]) ) kputc('.', str);
+ else kputw(bcf_gt_allele(ptr[i]), str);
+ }
+ if ( i==0 ) kputc('.', str);
+ if ( str->l - plen > blen )
+ {
+ // too many alternate alleles or ploidy is too large, the genotype does not fit
+ // three characters ("0/0" vs "10/10").
+ blen *= 2;
+ goto gt_length_too_big;
+ }
+ plen = str->l - plen;
+ while ( plen < blen )
+ {
+ kputc_(0, str);
+ plen++;
+ }
+ }
+ return 0;
+}
static int vcf_setter_format_gt(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
bcf1_t *rec = (bcf1_t*) data;
if ( nsrc==-3 ) return 0; // the tag is not present
if ( nsrc<=0 ) return 1; // error
+ // Genotypes are internally represented as integers. This is a complication when
+ // adding as a different Type=String field, such as FMT/newGT:=GT
+ if ( strcmp(col->hdr_key_src,col->hdr_key_dst) )
+ {
+ int nsmpl_dst = bcf_hdr_nsamples(args->hdr_out);
+ int nsmpl_src = bcf_hdr_nsamples(args->files->readers[1].header);
+ genotypes_to_string(args,nsrc/nsmpl_src,args->tmpi,nsmpl_dst,&args->tmpks);
+ return bcf_update_format_char(args->hdr_out,line,col->hdr_key_dst,args->tmpks.s,args->tmpks.l);
+ }
+
if ( !args->sample_map )
return bcf_update_genotypes(args->hdr_out,line,args->tmpi,nsrc);
}
static int setter_format_int(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with FORMAT tags (yet?)\n");
+
annot_line_t *tab = (annot_line_t*) data;
if ( col->icol+args->nsmpl_annot > tab->ncols )
- error("Incorrect number of values for %s at %s:%d\n",col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values for %s at %s:%"PRId64"\n",col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int nvals = count_vals(tab,col->icol,col->icol+args->nsmpl_annot);
hts_expand(int32_t,nvals*args->nsmpl_annot,args->mtmpi,args->tmpi);
char *end = str;
ptr[ival] = strtol(str, &end, 10);
if ( end==str )
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
ival++;
str = *end ? end+1 : end;
}
static int setter_format_real(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with FORMAT tags (yet?)\n");
+
annot_line_t *tab = (annot_line_t*) data;
if ( col->icol+args->nsmpl_annot > tab->ncols )
- error("Incorrect number of values for %s at %s:%d\n",col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values for %s at %s:%"PRId64"\n",col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int nvals = count_vals(tab,col->icol,col->icol+args->nsmpl_annot);
hts_expand(float,nvals*args->nsmpl_annot,args->mtmpf,args->tmpf);
char *end = str;
ptr[ival] = strtod(str, &end);
if ( end==str )
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
ival++;
str = *end ? end+1 : end;
}
static int setter_format_str(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with FORMAT tags (yet?)\n");
+
annot_line_t *tab = (annot_line_t*) data;
if ( col->icol+args->nsmpl_annot > tab->ncols )
- error("Incorrect number of values for %s at %s:%d\n",col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values for %s at %s:%"PRId64"\n",col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ismpl;
for (ismpl=0; ismpl<args->nsmpl_annot; ismpl++)
// create mapping from src to dst genotypes, haploid and diploid version
int nmap_hap = col->number==BCF_VL_G || col->number==BCF_VL_R ? rec->n_allele : rec->n_allele - 1;
int *map_hap = vcmp_map_ARvalues(args->vcmp,nmap_hap,line->n_allele,line->d.allele,rec->n_allele,rec->d.allele);
- if ( !map_hap ) error("REF alleles not compatible at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !map_hap ) error("REF alleles not compatible at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int i, j;
if ( rec->n_allele==line->n_allele )
}
int pld_src = determine_ploidy(rec->n_allele, args->tmpi, nsrc1, args->src_smpl_pld, nsmpl_src);
if ( pld_src<0 )
- error("Unexpected number of %s values (%d) for %d alleles at %s:%d\n", col->hdr_key_src,-pld_src, rec->n_allele, bcf_seqname(bcf_sr_get_header(args->files,1),rec),rec->pos+1);
+ error("Unexpected number of %s values (%d) for %d alleles at %s:%"PRId64"\n", col->hdr_key_src,-pld_src, rec->n_allele, bcf_seqname(bcf_sr_get_header(args->files,1),rec),(int64_t) rec->pos+1);
int pld_dst = determine_ploidy(line->n_allele, args->tmpi2, ndst1, args->dst_smpl_pld, nsmpl_dst);
if ( pld_dst<0 )
- error("Unexpected number of %s values (%d) for %d alleles at %s:%d\n", col->hdr_key_src,-pld_dst, line->n_allele, bcf_seqname(args->hdr,line),line->pos+1);
+ error("Unexpected number of %s values (%d) for %d alleles at %s:%"PRId64"\n", col->hdr_key_src,-pld_dst, line->n_allele, bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ndst1_new = pld_dst==1 ? line->n_allele : line->n_allele*(line->n_allele+1)/2;
if ( ndst1_new != ndst1 )
{
- if ( ndst1 ) error("todo: %s ndst1!=ndst .. %d %d at %s:%d\n",col->hdr_key_src,ndst1_new,ndst1,bcf_seqname(args->hdr,line),line->pos+1);
+ if ( ndst1 ) error("todo: %s ndst1!=ndst .. %d %d at %s:%"PRId64"\n",col->hdr_key_src,ndst1_new,ndst1,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
ndst1 = ndst1_new;
hts_expand(int32_t, ndst1*nsmpl_dst, args->mtmpi2, args->tmpi2);
}
if ( col->number==BCF_VL_G )
{
if ( args->src_smpl_pld[ii] > 0 && args->dst_smpl_pld[i] > 0 && args->src_smpl_pld[ii]!=args->dst_smpl_pld[i] )
- error("Sample ploidy differs at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ error("Sample ploidy differs at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
if ( !args->dst_smpl_pld[i] )
for (j=0; j<ndst1; j++) ptr_dst[j] = bcf_int32_missing;
}
}
static int vcf_setter_format_real(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
-
bcf1_t *rec = (bcf1_t*) data;
int nsrc = bcf_get_format_float(args->files->readers[1].header,rec,col->hdr_key_src,&args->tmpf,&args->mtmpf);
if ( nsrc==-3 ) return 0; // the tag is not present
// create mapping from src to dst genotypes, haploid and diploid version
int nmap_hap = col->number==BCF_VL_G || col->number==BCF_VL_R ? rec->n_allele : rec->n_allele - 1;
int *map_hap = vcmp_map_ARvalues(args->vcmp,nmap_hap,line->n_allele,line->d.allele,rec->n_allele,rec->d.allele);
- if ( !map_hap ) error("REF alleles not compatible at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !map_hap ) error("REF alleles not compatible at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int i, j;
if ( rec->n_allele==line->n_allele )
}
int pld_src = determine_ploidy(rec->n_allele, args->tmpi, nsrc1, args->src_smpl_pld, nsmpl_src);
if ( pld_src<0 )
- error("Unexpected number of %s values (%d) for %d alleles at %s:%d\n", col->hdr_key_src,-pld_src, rec->n_allele, bcf_seqname(bcf_sr_get_header(args->files,1),rec),rec->pos+1);
+ error("Unexpected number of %s values (%d) for %d alleles at %s:%"PRId64"\n", col->hdr_key_src,-pld_src, rec->n_allele, bcf_seqname(bcf_sr_get_header(args->files,1),rec),(int64_t) rec->pos+1);
int pld_dst = determine_ploidy(line->n_allele, args->tmpi2, ndst1, args->dst_smpl_pld, nsmpl_dst);
if ( pld_dst<0 )
- error("Unexpected number of %s values (%d) for %d alleles at %s:%d\n", col->hdr_key_src,-pld_dst, line->n_allele, bcf_seqname(args->hdr,line),line->pos+1);
+ error("Unexpected number of %s values (%d) for %d alleles at %s:%"PRId64"\n", col->hdr_key_src,-pld_dst, line->n_allele, bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ndst1_new = pld_dst==1 ? line->n_allele : line->n_allele*(line->n_allele+1)/2;
if ( ndst1_new != ndst1 )
{
- if ( ndst1 ) error("todo: %s ndst1!=ndst .. %d %d at %s:%d\n",col->hdr_key_src,ndst1_new,ndst1,bcf_seqname(args->hdr,line),line->pos+1);
+ if ( ndst1 ) error("todo: %s ndst1!=ndst .. %d %d at %s:%"PRId64"\n",col->hdr_key_src,ndst1_new,ndst1,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
ndst1 = ndst1_new;
hts_expand(float, ndst1*nsmpl_dst, args->mtmpf2, args->tmpf2);
}
if ( col->number==BCF_VL_G )
{
if ( args->src_smpl_pld[ii] > 0 && args->dst_smpl_pld[i] > 0 && args->src_smpl_pld[ii]!=args->dst_smpl_pld[i] )
- error("Sample ploidy differs at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ error("Sample ploidy differs at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
if ( !args->dst_smpl_pld[i] )
for (j=0; j<ndst1; j++) bcf_float_set_missing(ptr_dst[j]);
}
args->tmps = args->tmpp[0]; // tmps might be realloced
if ( ret==-3 ) return 0; // the tag is not present
if ( ret<=0 ) return 1; // error
- return core_setter_format_str(args,line,col,args->tmpp);
+ if ( strcmp("GT",col->hdr_key_dst) )
+ return core_setter_format_str(args,line,col,args->tmpp);
+
+ // Genotypes are internally represented as integers. This is a complication for FMT/GT:=oldGT
+ // First determine the maximum number of alleles per-sample ndst1
+ int nsmpl_src = bcf_hdr_nsamples(args->files->readers[1].header);
+ int nsmpl_dst = bcf_hdr_nsamples(args->hdr_out);
+ int isrc,idst, ndst1 = 0, nsrc1 = ret / nsmpl_src;
+ char *ptr = args->tmps, *ptr_end = ptr + ret;
+ while ( ptr < ptr_end )
+ {
+ char *smpl_end = ptr + nsrc1;
+ int n = 1;
+ while ( ptr < smpl_end )
+ {
+ if ( *ptr=='/' || *ptr=='|' ) n++;
+ ptr++;
+ }
+ if ( ndst1 < n ) ndst1 = n;
+ }
+ assert( ndst1 );
+
+ int ndst = ndst1*nsmpl_dst;
+ hts_expand(int32_t,ndst,args->mtmpi,args->tmpi);
+ hts_expand(char,ret+1,args->mtmps,args->tmps); args->tmps[ret] = 0; // the FORMAT string may not be 0-terminated
+ for (idst=0; idst<nsmpl_dst; idst++)
+ {
+ int i = 0, is_phased = 0;
+ int32_t *dst = args->tmpi + idst*ndst1;
+ isrc = args->sample_map ? args->sample_map[idst] : idst;
+ if ( isrc==-1 )
+ {
+ dst[0] = bcf_gt_missing;
+ for (i=1; i<ndst1; i++) dst[i] = bcf_int32_vector_end;
+ continue;
+ }
+ char *beg = args->tmps + isrc*nsrc1, *tmp;
+ char *keep_ptr = beg+nsrc1, keep = *keep_ptr; *keep_ptr = 0;
+ while ( *beg )
+ {
+ char *end = beg;
+ while ( *end && *end!='/' && *end!='|' ) end++;
+ if ( *beg=='.' && end-beg==1 ) dst[i] = bcf_gt_missing;
+ else
+ {
+ if ( *end=='|' ) is_phased = 1;
+ dst[i] = strtol(beg, &tmp, 10);
+ if ( tmp!=end )
+ error("Could not parse the %s field at %s:%"PRId64" in %s\n", col->hdr_key_src,bcf_seqname(args->files->readers[1].header,rec),(int64_t) rec->pos+1,args->targets_fname);
+ if ( dst[i] >= line->n_allele )
+ error("The source allele index is bigger than the number of destination alleles at %s:%"PRId64"\n", bcf_seqname(args->files->readers[1].header,rec),(int64_t) rec->pos+1);
+ dst[i] = is_phased ? bcf_gt_phased(dst[i]) : bcf_gt_unphased(dst[i]);
+ }
+ beg = *end ? end+1 : end;
+ i++;
+ }
+ *keep_ptr = keep;
+ for (; i<ndst1; i++) dst[i] = bcf_int32_vector_end;
+ }
+ return bcf_update_genotypes(args->hdr_out,line,args->tmpi,ndst);
}
static int init_sample_map(args_t *args, bcf_hdr_t *src, bcf_hdr_t *dst)
{
args->sample_map = (int*) malloc(sizeof(int)*args->nsample_map);
for (i=0; i<args->nsample_map; i++) args->sample_map[i] = -1;
- // possible todo: could do with smpl_ilist only
- smpl_ilist_t *ilist = smpl_ilist_init(dst, args->sample_names, args->sample_is_file, SMPL_STRICT);
- if ( !ilist || !ilist->n ) error("Could not parse: %s\n", args->sample_names);
- char **samples = (char**) malloc(sizeof(char*)*ilist->n);
- for (i=0; i<ilist->n; i++) samples[i] = strdup(dst->samples[i]);
+ int flags = !src ? SMPL_STRICT|SMPL_SINGLE : SMPL_STRICT|SMPL_SINGLE|SMPL_PAIR2; // is vcf vs tab annotation file
+ smpl_ilist_t *ilist = smpl_ilist_init(dst, args->sample_names, args->sample_is_file, flags); // gives mapping dst->src
+ if ( !ilist || !ilist->n ) error("Could not parse the samples: %s\n", args->sample_names);
args->nsmpl_annot = ilist->n;
- smpl_ilist_destroy(ilist);
int need_sample_map = args->nsmpl_annot==bcf_hdr_nsamples(dst) ? 0 : 1;
- if ( !src )
- {
- // tab annotation file
- for (i=0; i<args->nsmpl_annot; i++)
- {
- int idst = bcf_hdr_id2int(dst, BCF_DT_SAMPLE, samples[i]);
- if ( idst==-1 ) error("Sample \"%s\" not found in the destination file\n", samples[i]);
- args->sample_map[idst] = i;
- if ( idst!=i ) need_sample_map = 1;
- }
- }
- else
+ for (i=0; i<args->nsmpl_annot; i++)
{
- // vcf annotation file
- for (i=0; i<args->nsmpl_annot; i++)
+ int idst = ilist->idx[i];
+ const char *src_name = ilist->pair && ilist->pair[i] ? ilist->pair[i] : bcf_hdr_int2id(dst, BCF_DT_SAMPLE, idst);
+ int isrc = i;
+ if ( src ) // the annotation file is a VCF, not a tab-delimited file
{
- int isrc, idst;
- char *ss = samples[i], *se = samples[i];
- while ( *se && !isspace(*se) ) se++;
- if ( !*se )
- {
- // only one sample name
- isrc = bcf_hdr_id2int(src, BCF_DT_SAMPLE,ss);
- if ( isrc==-1 ) error("Sample \"%s\" not found in the source file\n", ss);
- idst = bcf_hdr_id2int(dst, BCF_DT_SAMPLE,ss);
- if ( idst==-1 ) error("Sample \"%s\" not found in the destination file\n", ss);
- args->sample_map[idst] = isrc;
- if ( idst!=isrc ) need_sample_map = 1;
- continue;
- }
- *se = 0;
- isrc = bcf_hdr_id2int(src, BCF_DT_SAMPLE,ss);
- if ( isrc==-1 ) error("Sample \"%s\" not found in the source file\n", ss);
-
- ss = se+1;
- while ( isspace(*ss) ) ss++;
- se = ss;
- while ( *se && !isspace(*se) ) se++;
-
- idst = bcf_hdr_id2int(dst, BCF_DT_SAMPLE,ss);
- if ( idst==-1 ) error("Sample \"%s\" not found in the destination file\n", ss);
-
- args->sample_map[idst] = isrc;
- if ( idst!=isrc ) need_sample_map = 1;
+ isrc = bcf_hdr_id2int(src, BCF_DT_SAMPLE, src_name);
+ if ( isrc==-1 ) error("Sample \"%s\" not found in the annotation file\n", src_name);
}
+ if ( isrc!=idst ) need_sample_map = 1;
+ args->sample_map[idst] = isrc;
}
- for (i=0; i<args->nsmpl_annot; i++) free(samples[i]);
- free(samples);
+ smpl_ilist_destroy(ilist);
return need_sample_map;
}
static char *columns_complement(char *columns, void **skip_info, void **skip_fmt)
kputsn(ss, se-ss, &str);
if ( !str.s[0] || !strcasecmp("-",str.s) ) ;
else if ( !strcasecmp("CHROM",str.s) ) args->chr_idx = icol;
- else if ( !strcasecmp("POS",str.s) ) args->from_idx = icol;
- else if ( !strcasecmp("FROM",str.s) ) args->from_idx = icol;
- else if ( !strcasecmp("TO",str.s) ) args->to_idx = icol;
+ else if ( !strcasecmp("POS",str.s) ) args->beg_idx = icol;
+ else if ( !strcasecmp("FROM",str.s) || !strcasecmp("BEG",str.s) ) args->beg_idx = icol;
+ else if ( !strcasecmp("TO",str.s) || !strcasecmp("END",str.s) ) args->end_idx = icol;
else if ( !strcasecmp("REF",str.s) )
{
if ( args->tgts_is_vcf )
bcf_hrec_format(hrec, &tmp);
bcf_hdr_append(args->hdr_out, tmp.s);
}
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
}
}
else if ( !strcasecmp("QUAL",str.s) )
tmp.l = 0;
bcf_hrec_format(hrec, &tmp);
bcf_hdr_append(args->hdr_out, tmp.s);
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
int hdr_id = bcf_hdr_id2int(args->hdr_out, BCF_DT_ID, hrec->vals[k]);
args->ncols++; args->cols = (annot_col_t*) realloc(args->cols,sizeof(annot_col_t)*args->ncols);
annot_col_t *col = &args->cols[args->ncols-1];
tmp.l = 0;
bcf_hrec_format(hrec, &tmp);
bcf_hdr_append(args->hdr_out, tmp.s);
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
int hdr_id = bcf_hdr_id2int(args->hdr_out, BCF_DT_ID, hrec->vals[k]);
args->ncols++; args->cols = (annot_col_t*) realloc(args->cols,sizeof(annot_col_t)*args->ncols);
annot_col_t *col = &args->cols[args->ncols-1];
tmp.l = 0;
bcf_hrec_format_rename(hrec, key_dst, &tmp);
bcf_hdr_append(args->hdr_out, tmp.s);
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
}
int hdr_id = bcf_hdr_id2int(args->hdr_out, BCF_DT_ID, key_dst);
if ( !bcf_hdr_idinfo_exists(args->hdr_out,BCF_HL_FMT,hdr_id) )
{
if ( replace==REPLACE_NON_MISSING ) error("Apologies, the -INFO/TAG feature has not been implemented yet.\n");
if ( replace==SET_OR_APPEND ) error("Apologies, the =INFO/TAG feature has not been implemented yet.\n");
- char *key_dst = !strncasecmp("INFO/",str.s,5) ? str.s + 5 : str.s;
+ int explicit_info = 0;
+ char *key_dst;
+ if ( !strncasecmp("INFO/",str.s,5) )
+ {
+ key_dst = str.s + 5;
+ explicit_info = 1;
+ }
+ else
+ key_dst = str.s;
char *key_src = strstr(key_dst,":=");
if ( key_src )
{
*key_src = 0;
key_src += 2;
- if ( !strncasecmp("INFO/",key_src,5) ) key_src += 5;
+ if ( !strncasecmp("INFO/",key_src,5) )
+ {
+ key_src += 5;
+ explicit_info = 1;
+ }
+ else if ( !strncasecmp("FMT/",key_src,4) || !strncasecmp("FORMAT/",key_src,5) )
+ {
+ key_src[-2] = ':';
+ error("Did you mean \"FMT/%s\" rather than \"%s\"?\n",str.s,str.s);
+ }
}
else
key_src = key_dst;
if ( args->tgts_is_vcf ) // reading annotations from a VCF, add a new header line
{
bcf_hrec_t *hrec = bcf_hdr_get_hrec(args->files->readers[1].header, BCF_HL_INFO, "ID", key_src, NULL);
- if ( !hrec ) error("The tag \"%s\" is not defined in %s\n", str.s,args->files->readers[1].fname);
+ if ( !hrec )
+ {
+ if ( !explicit_info && bcf_hdr_get_hrec(args->files->readers[1].header, BCF_HL_FMT, "ID", key_src, NULL) )
+ error("Did you mean \"FMT/%s\" rather than \"%s\"?\n",str.s,str.s);
+ fprintf(stderr,"[%s] %d\n",key_src,explicit_info);
+ error("The tag \"%s\" is not defined in %s\n", key_src,args->files->readers[1].fname);
+ }
tmp.l = 0;
bcf_hrec_format_rename(hrec, key_dst, &tmp);
bcf_hdr_append(args->hdr_out, tmp.s);
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
hdr_id = bcf_hdr_id2int(args->hdr_out, BCF_DT_ID, key_dst);
}
else
}
free(str.s);
free(tmp.s);
- if ( args->to_idx==-1 ) args->to_idx = args->from_idx;
free(args->columns);
if ( skip_info ) khash_str2int_destroy_free(skip_info);
if ( skip_fmt ) khash_str2int_destroy_free(skip_fmt);
else if ( sample_map_ok<0 )
error("No matching samples in source and destination file?\n");
}
+static void init_merge_method(args_t *args)
+{
+ int i;
+ for (i=0; i<args->ncols; i++)
+ {
+ args->cols[i].merge_method = MM_FIRST;
+ args->cols[i].mm_str_hash = NULL;
+ args->cols[i].mm_dbl = NULL;
+ args->cols[i].mm_dbl_nalloc = args->cols[i].mm_dbl_nused = args->cols[i].mm_dbl_ndat = 0;
+ memset(&args->cols[i].mm_kstr, 0, sizeof(args->cols[i].mm_kstr));
+ }
+ if ( !args->merge_method_str ) return;
+ if ( args->tgts_is_vcf ) error("Error: the --merge-logic is intended for use with BED or TAB-delimited files only.\n");
+ if ( !args->tgt_idx ) error("Error: BEG,END (or FROM,TO) columns are expected with the --merge-logic option.\n");
+ char *sb = args->merge_method_str;
+ while ( *sb )
+ {
+ char *se = sb;
+ while ( *se && *se!=',' ) se++;
+ args->tmpks.l = 0;
+ kputsn(sb, se-sb, &args->tmpks);
+ kputc(0, &args->tmpks);
+ char *mm_type_str = args->tmpks.s + args->tmpks.l;
+ while ( *mm_type_str!=':' && mm_type_str > args->tmpks.s ) mm_type_str--;
+ if ( *mm_type_str!=':' )
+ error("Error: could not parse the argument to --merge-logic: %s\n", args->merge_method_str);
+ *mm_type_str = 0;
+ mm_type_str++;
+ int mm_type = MM_FIRST;
+ if ( !strcasecmp("unique",mm_type_str) ) mm_type = MM_UNIQUE;
+ else if ( !strcasecmp("append",mm_type_str) ) mm_type = MM_APPEND;
+ else if ( !strcasecmp("sum",mm_type_str) ) mm_type = MM_SUM;
+ else if ( !strcasecmp("avg",mm_type_str) ) mm_type = MM_AVG;
+ else if ( !strcasecmp("min",mm_type_str) ) mm_type = MM_MIN;
+ else if ( !strcasecmp("max",mm_type_str) ) mm_type = MM_MAX;
+ else error("Error: could not parse --merge-logic %s, the logic \"%s\" is not recognised\n", args->merge_method_str,mm_type_str);
+ for (i=0; i<args->ncols; i++)
+ {
+ if ( strcmp(args->cols[i].hdr_key_dst,args->tmpks.s) ) continue;
+ if ( mm_type==MM_APPEND && args->cols[i].number!=BCF_VL_VAR )
+ error("Error: --merge-logic append can be requested only for tags of variable length (Number=.)\n");
+ args->cols[i].merge_method = mm_type;
+ break;
+ }
+ if ( i==args->ncols ) error("No such tag in the destination file: %s\n", args->tmpks.s);
+ sb = *se ? se + 1 : se;
+ }
+}
static void rename_chrs(args_t *args, char *fname)
{
{
if ( !args->columns ) error("The -c option not given\n");
if ( args->chr_idx==-1 ) error("The -c CHROM option not given\n");
- if ( args->from_idx==-1 ) error("The -c POS option not given\n");
- if ( args->to_idx==-1 ) args->to_idx = -args->from_idx - 1;
-
- args->tgts = bcf_sr_regions_init(args->targets_fname,1,args->chr_idx,args->from_idx,args->to_idx);
- if ( !args->tgts ) error("Could not initialize the annotation file: %s\n", args->targets_fname);
- if ( !args->tgts->tbx ) error("Expected tabix-indexed annotation file: %s\n", args->targets_fname);
+ if ( args->beg_idx==-1 ) error("The -c POS option not given\n");
+ if ( args->single_overlaps && args->merge_method_str ) error("The options --merge-logic and --single-overlaps cannot be combined\n");
+ if ( args->end_idx==-1 || (args->single_overlaps && !args->merge_method_str) )
+ {
+ args->end_idx = -args->beg_idx - 1;
+ args->tgts = bcf_sr_regions_init(args->targets_fname,1,args->chr_idx,args->beg_idx,args->end_idx);
+ if ( !args->tgts ) error("Could not initialize the annotation file: %s\n", args->targets_fname);
+ if ( !args->tgts->tbx ) error("Expected tabix-indexed annotation file: %s\n", args->targets_fname);
+ }
+ else
+ {
+ if ( args->ref_idx!=-1 ) error("Error: the REF columns will be ignored when BEG,END (or FROM,TO) is present. Replace END (or TO) with \"-\".\n");
+ int len = strlen(args->targets_fname);
+ if ( len>=7 && !strcasecmp(".bed.gz",args->targets_fname+len-7) ) args->tgt_is_bed = 1;
+ else if ( len>=8 && !strcasecmp(".bed.bgz",args->targets_fname+len-8) ) args->tgt_is_bed = 1;
+ else if ( len>=4 && !strcasecmp(".bed",args->targets_fname+len-4) ) args->tgt_is_bed = 1;
+ args->tgt_idx = regidx_init(args->targets_fname,parse_with_payload,free_payload,sizeof(char*),args);
+ if ( !args->tgt_idx ) error("Failed to parse: %s\n", args->targets_fname);
+ args->tgt_itr = regitr_init(args->tgt_idx);
+ args->nalines++;
+ hts_expand0(annot_line_t,args->nalines,args->malines,args->alines);
+ }
}
+ init_merge_method(args);
args->vcmp = vcmp_init();
if ( args->filter_str )
if ( args->rename_chrs ) rename_chrs(args, args->rename_chrs);
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
+ if ( args->out_fh == NULL ) error("[%s] Error: cannot write to \"%s\": %s\n", __func__,args->output_fname, strerror(errno));
if ( args->n_threads )
hts_set_opt(args->out_fh, HTS_OPT_THREAD_POOL, args->files->p);
- bcf_hdr_write(args->out_fh, args->hdr_out);
+ if ( bcf_hdr_write(args->out_fh, args->hdr_out)!=0 ) error("[%s] Error: failed to write the header to %s\n", __func__,args->output_fname);
}
}
{
free(args->cols[i].hdr_key_src);
free(args->cols[i].hdr_key_dst);
+ free(args->cols[i].mm_kstr.s);
+ if ( args->cols[i].mm_str_hash ) khash_str2int_destroy_free(args->cols[i].mm_str_hash);
+ free(args->cols[i].mm_dbl);
}
free(args->cols);
for (i=0; i<args->malines; i++)
free(args->alines[i].line.s);
}
free(args->alines);
+ if ( args->tgt_idx )
+ {
+ regidx_destroy(args->tgt_idx);
+ regitr_destroy(args->tgt_itr);
+ }
if ( args->tgts ) bcf_sr_regions_destroy(args->tgts);
free(args->tmpks.s);
free(args->tmpi);
free(args->sample_map);
}
+static void parse_annot_line(args_t *args, char *str, annot_line_t *tmp)
+{
+ tmp->line.l = 0;
+ kputs(str, &tmp->line);
+ char *s = tmp->line.s;
+ tmp->ncols = 1;
+ hts_expand(char*,tmp->ncols,tmp->mcols,tmp->cols);
+ tmp->cols[0] = s;
+ while ( *s )
+ {
+ if ( *s=='\t' )
+ {
+ tmp->ncols++;
+ hts_expand(char*,tmp->ncols,tmp->mcols,tmp->cols);
+ tmp->cols[tmp->ncols-1] = s+1;
+ *s = 0;
+ }
+ s++;
+ }
+ if ( args->ref_idx != -1 )
+ {
+ if ( args->ref_idx >= tmp->ncols )
+ error("Could not parse the line, expected %d+ columns, found %d:\n\t%s\n",args->ref_idx+1,tmp->ncols,str);
+ if ( args->alt_idx >= tmp->ncols )
+ error("Could not parse the line, expected %d+ columns, found %d:\n\t%s\n",args->alt_idx+1,tmp->ncols,str);
+ tmp->nals = 2;
+ hts_expand(char*,tmp->nals,tmp->mals,tmp->als);
+ tmp->als[0] = tmp->cols[args->ref_idx];
+ tmp->als[1] = s = tmp->cols[args->alt_idx];
+ while ( *s )
+ {
+ if ( *s==',' )
+ {
+ tmp->nals++;
+ hts_expand(char*,tmp->nals,tmp->mals,tmp->als);
+ tmp->als[tmp->nals-1] = s+1;
+ *s = 0;
+ }
+ s++;
+ }
+ }
+}
static void buffer_annot_lines(args_t *args, bcf1_t *line, int start_pos, int end_pos)
{
if ( args->nalines && args->alines[0].rid != line->rid ) args->nalines = 0;
tmp->rid = line->rid;
tmp->start = args->tgts->start;
tmp->end = args->tgts->end;
- tmp->line.l = 0;
- kputs(args->tgts->line.s, &tmp->line);
- char *s = tmp->line.s;
- tmp->ncols = 1;
- hts_expand(char*,tmp->ncols,tmp->mcols,tmp->cols);
- tmp->cols[0] = s;
- while ( *s )
- {
- if ( *s=='\t' )
- {
- tmp->ncols++;
- hts_expand(char*,tmp->ncols,tmp->mcols,tmp->cols);
- tmp->cols[tmp->ncols-1] = s+1;
- *s = 0;
- }
- s++;
- }
+ parse_annot_line(args, args->tgts->line.s, tmp);
if ( args->ref_idx != -1 )
{
- if ( args->ref_idx >= tmp->ncols )
- error("Could not parse the line, expected %d+ columns, found %d:\n\t%s\n",args->ref_idx+1,tmp->ncols,args->tgts->line.s);
- if ( args->alt_idx >= tmp->ncols )
- error("Could not parse the line, expected %d+ columns, found %d:\n\t%s\n",args->alt_idx+1,tmp->ncols,args->tgts->line.s);
- tmp->nals = 2;
- hts_expand(char*,tmp->nals,tmp->mals,tmp->als);
- tmp->als[0] = tmp->cols[args->ref_idx];
- tmp->als[1] = s = tmp->cols[args->alt_idx];
- while ( *s )
- {
- if ( *s==',' )
- {
- tmp->nals++;
- hts_expand(char*,tmp->nals,tmp->mals,tmp->als);
- tmp->als[tmp->nals-1] = s+1;
- *s = 0;
- }
- s++;
- }
int iseq = args->tgts->iseq;
if ( bcf_sr_regions_next(args->tgts)<0 || args->tgts->iseq!=iseq ) break;
}
for (i=0; i<args->nrm; i++)
args->rm[i].handler(args, line, &args->rm[i]);
- if ( args->tgts )
+ int has_overlap = 0;
+
+ if ( args->tgt_idx )
+ {
+ if ( regidx_overlap(args->tgt_idx, bcf_seqname(args->hdr,line),line->pos,line->pos+line->rlen-1, args->tgt_itr) )
+ {
+ while ( regitr_overlap(args->tgt_itr) )
+ {
+ annot_line_t *tmp = &args->alines[0];
+ tmp->rid = line->rid;
+ tmp->start = args->tgt_itr->beg;
+ tmp->end = args->tgt_itr->end;
+ parse_annot_line(args, regitr_payload(args->tgt_itr,char*), tmp);
+ for (j=0; j<args->ncols; j++)
+ if ( args->cols[j].setter(args,line,&args->cols[j],tmp) )
+ error("fixme: Could not set %s at %s:%"PRId64"\n", args->cols[j].hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
+ }
+ has_overlap = 1;
+ }
+ for (j=0; j<args->ncols; j++)
+ if ( args->cols[j].merge_method != MM_FIRST )
+ args->cols[j].setter(args,line,&args->cols[j],NULL);
+ }
+ else if ( args->tgts )
{
// Buffer annotation lines. When multiple ALT alleles are present in the
// annotation file, at least one must match one of the VCF alleles.
// there is a matching line
for (j=0; j<args->ncols; j++)
if ( args->cols[j].setter(args,line,&args->cols[j],&args->alines[i]) )
- error("fixme: Could not set %s at %s:%d\n", args->cols[j].hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
-
- }
-
- if ( args->mark_sites )
- {
- // ideally, we'd like to be far more general than this in future, see https://github.com/samtools/bcftools/issues/87
- if ( args->mark_sites_logic==MARK_LISTED )
- bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL,i<args->nalines?1:0);
- else
- bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL,i<args->nalines?0:1);
+ error("fixme: Could not set %s at %s:%"PRId64"\n", args->cols[j].hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
}
+ has_overlap = i<args->nalines ? 1 : 0;
}
else if ( args->files->nreaders == 2 )
{
bcf1_t *aline = bcf_sr_get_line(args->files,1);
for (j=0; j<args->ncols; j++)
if ( args->cols[j].setter(args,line,&args->cols[j],aline) )
- error("fixme: Could not set %s at %s:%d\n", args->cols[j].hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("fixme: Could not set %s at %s:%"PRId64"\n", args->cols[j].hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
- if ( args->mark_sites )
- bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL,args->mark_sites_logic==MARK_LISTED ? 1 : 0);
+ has_overlap = 1;
}
- else if ( args->mark_sites )
- bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL, args->mark_sites_logic==MARK_UNLISTED ? 1 : 0);
}
if ( args->set_ids )
{
bcf_update_id(args->hdr_out,line,args->tmpks.s);
}
}
+
+ if ( args->mark_sites )
+ {
+ // ideally, we'd like to be far more general than this in future, see https://github.com/samtools/bcftools/issues/87
+ if ( args->mark_sites_logic==MARK_LISTED )
+ bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL,has_overlap?1:0);
+ else
+ bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL,has_overlap?0:1);
+ }
}
static void usage(args_t *args)
fprintf(stderr, " --collapse <string> matching records by <snps|indels|both|all|some|none>, see man page for details [some]\n");
fprintf(stderr, " -c, --columns <list> list of columns in the annotation file, e.g. CHROM,POS,REF,ALT,-,INFO/TAG. See man page for details\n");
fprintf(stderr, " -e, --exclude <expr> exclude sites for which the expression is true (see man page for details)\n");
+ fprintf(stderr, " --force continue despite parsing error (at your own risk!)\n");
fprintf(stderr, " -h, --header-lines <file> lines which should be appended to the VCF header\n");
fprintf(stderr, " -I, --set-id [+]<format> set ID column, see man page for details\n");
fprintf(stderr, " -i, --include <expr> select sites for which the expression is true (see man page for details)\n");
fprintf(stderr, " -k, --keep-sites leave -i/-e sites unchanged instead of discarding them\n");
+ fprintf(stderr, " -l, --merge-logic <tag:type> merge logic for multiple overlapping regions (see man page for details), EXPERIMENTAL\n");
fprintf(stderr, " -m, --mark-sites [+-]<tag> add INFO/tag flag to sites which are (\"+\") or are not (\"-\") listed in the -a file\n");
fprintf(stderr, " --no-version do not append version and command line to the header\n");
fprintf(stderr, " -o, --output <file> write output to a file [standard output]\n");
fprintf(stderr, " --rename-chrs <file> rename sequences according to map file: from\\tto\n");
fprintf(stderr, " -s, --samples [^]<list> comma separated list of samples to annotate (or exclude with \"^\" prefix)\n");
fprintf(stderr, " -S, --samples-file [^]<file> file of samples to annotate (or exclude with \"^\" prefix)\n");
+ fprintf(stderr, " --single-overlaps keep memory low by avoiding complexities arising from handling multiple overlapping intervals\n");
fprintf(stderr, " -x, --remove <list> list of annotations (e.g. ID,INFO/DP,FORMAT/DP,FILTER) to remove (or keep with \"^\" prefix). See man page for details\n");
fprintf(stderr, " --threads <int> number of extra output compression threads [0]\n");
fprintf(stderr, "\n");
args->output_type = FT_VCF;
args->n_threads = 0;
args->record_cmd_line = 1;
- args->ref_idx = args->alt_idx = args->chr_idx = args->from_idx = args->to_idx = -1;
+ args->ref_idx = args->alt_idx = args->chr_idx = args->beg_idx = args->end_idx = -1;
args->set_ids_replace = 1;
int regions_is_file = 0, collapse = 0;
static struct option loptions[] =
{
- {"keep-sites",required_argument,NULL,'k'},
+ {"keep-sites",no_argument,NULL,'k'},
{"mark-sites",required_argument,NULL,'m'},
{"set-id",required_argument,NULL,'I'},
{"output",required_argument,NULL,'o'},
{"output-type",required_argument,NULL,'O'},
{"threads",required_argument,NULL,9},
{"annotations",required_argument,NULL,'a'},
+ {"merge-logic",required_argument,NULL,'l'},
{"collapse",required_argument,NULL,2},
{"include",required_argument,NULL,'i'},
{"exclude",required_argument,NULL,'e'},
{"header-lines",required_argument,NULL,'h'},
{"samples",required_argument,NULL,'s'},
{"samples-file",required_argument,NULL,'S'},
+ {"single-overlaps",no_argument,NULL,10},
{"no-version",no_argument,NULL,8},
+ {"force",no_argument,NULL,'f'},
{NULL,0,NULL,0}
};
- while ((c = getopt_long(argc, argv, "h:?o:O:r:R:a:x:c:i:e:S:s:I:m:k",loptions,NULL)) >= 0)
+ while ((c = getopt_long(argc, argv, "h:?o:O:r:R:a:x:c:i:e:S:s:I:m:kl:f",loptions,NULL)) >= 0)
{
switch (c) {
+ case 'f': args->force = 1; break;
case 'k': args->keep_sites = 1; break;
case 'm':
args->mark_sites_logic = MARK_LISTED;
else if ( optarg[0]=='-' ) { args->mark_sites = optarg+1; args->mark_sites_logic = MARK_UNLISTED; }
else args->mark_sites = optarg;
break;
+ case 'l': args->merge_method_str = optarg; break;
case 'I': args->set_ids_fmt = optarg; break;
case 's': args->sample_names = optarg; break;
case 'S': args->sample_names = optarg; args->sample_is_file = 1; break;
break;
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
+ case 10 : args->single_overlaps = 1; break;
case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
if ( args->targets_fname )
{
htsFile *fp = hts_open(args->targets_fname,"r");
+ if ( !fp ) error("Failed to open %s\n", args->targets_fname);
htsFormat type = *hts_get_format(fp);
hts_close(fp);
}
}
if ( bcf_sr_set_threads(args->files, args->n_threads)<0 ) error("Failed to create threads\n");
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
+ static int line_errcode_warned = 0;
init_data(args);
while ( bcf_sr_next_line(args->files) )
{
if ( !bcf_sr_has_line(args->files,0) ) continue;
bcf1_t *line = bcf_sr_get_line(args->files,0);
- if ( line->errcode ) error("Encountered error, cannot proceed. Please check the error output above.\n");
+ if ( line->errcode )
+ {
+ if ( !args->force )
+ error("Encountered an error, cannot proceed. Please check the error output above.\n"
+ "If feeling adventurous, use the --force option. (At your own risk!)\n");
+ else if ( !line_errcode_warned )
+ {
+ fprintf(stderr,
+ "Warning: Encountered an error, proceeding only because --force was given.\n"
+ " Note that this can result in a segfault or a silent corruption of the output file!\n");
+ line_errcode_warned = 1;
+ line->errcode = 0;
+ }
+ }
if ( args->filter )
{
int pass = filter_test(args->filter, line, NULL);
if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
if ( !pass )
{
- if ( args->keep_sites ) bcf_write1(args->out_fh, args->hdr_out, line);
+ if ( args->keep_sites && bcf_write1(args->out_fh, args->hdr_out, line)!=0 ) error("[%s] Error: failed to write to %s\n", __func__,args->output_fname);
continue;
}
}
annotate(args, line);
- bcf_write1(args->out_fh, args->hdr_out, line);
+ if ( bcf_write1(args->out_fh, args->hdr_out, line)!=0 ) error("[%s] Error: failed to write to %s\n", __func__,args->output_fname);
}
destroy_data(args);
bcf_sr_destroy(args->files);
/* vcfannotate.c -- Annotate and edit VCF/BCF files.
- Copyright (C) 2013-2018 Genome Research Ltd.
+ Copyright (C) 2013-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#include <sys/types.h>
#include <dirent.h>
#include <math.h>
+#include <inttypes.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/kseq.h>
#include <htslib/khash_str2int.h>
-#include <dlfcn.h>
#include "bcftools.h"
#include "vcmp.h"
#include "filter.h"
#include "convert.h"
#include "smpl_ilist.h"
+#include "regidx.h"
struct _args_t;
}
annot_line_t;
-#define REPLACE_MISSING 0 // replace only missing values
-#define REPLACE_ALL 1 // replace both missing and existing values
-#define REPLACE_NON_MISSING 2 // replace only if tgt is not missing
-#define SET_OR_APPEND 3 // set new value if missing or non-existent, append otherwise
+#define REPLACE_MISSING 0 // replace only missing values
+#define REPLACE_ALL 1 // replace both missing and existing values
+#define REPLACE_NON_MISSING 2 // replace only if tgt is not missing
+#define SET_OR_APPEND 3 // set new value if missing or non-existent, append otherwise
+#define MM_FIRST 0 // if multiple annotation lines overlap a VCF record, use the first, discarding the rest
+#define MM_APPEND 1 // append, possibly multiple times
+#define MM_UNIQUE 2 // append, only unique values
+#define MM_SUM 3
+#define MM_AVG 4
+#define MM_MIN 5
+#define MM_MAX 6
typedef struct _annot_col_t
{
int icol, replace, number; // number: one of BCF_VL_* types
char *hdr_key_src, *hdr_key_dst;
int (*setter)(struct _args_t *, bcf1_t *, struct _annot_col_t *, void*);
+ int merge_method; // one of the MM_* defines
+ khash_t(str2int) *mm_str_hash; // lookup table to ensure uniqueness of added string values
+ kstring_t mm_kstr;
+ double
+ mm_dbl_nalloc, // the allocated size --merge-logic values array
+ mm_dbl_nused, // the number of used elements in the mm_dbl array
+ mm_dbl_ndat, // the number of merged rows (for calculating the average)
+ *mm_dbl;
}
annot_col_t;
int output_type, n_threads;
bcf_sr_regions_t *tgts;
+ regidx_t *tgt_idx;
+ regitr_t *tgt_itr;
+ int tgt_is_bed;
+
filter_t *filter;
char *filter_str;
int filter_logic; // include or exclude sites which match the filters? One of FLT_INCLUDE/FLT_EXCLUDE
vcmp_t *vcmp; // for matching annotation and VCF lines by allele
annot_line_t *alines; // buffered annotation lines
int nalines, malines;
- int ref_idx, alt_idx, chr_idx, from_idx, to_idx; // -1 if not present
+ int ref_idx, alt_idx, chr_idx, beg_idx, end_idx; // -1 if not present
annot_col_t *cols; // column indexes and setters
int ncols;
char **argv, *output_fname, *targets_fname, *regions_list, *header_fname;
char *remove_annots, *columns, *rename_chrs, *sample_names, *mark_sites;
- int argc, drop_header, record_cmd_line, tgts_is_vcf, mark_sites_logic;
+ char *merge_method_str;
+ int argc, drop_header, record_cmd_line, tgts_is_vcf, mark_sites_logic, force, single_overlaps;
}
args_t;
char *msprintf(const char *fmt, ...);
+int parse_with_payload(const char *line, char **chr_beg, char **chr_end, uint32_t *beg, uint32_t *end, void *payload, void *usr)
+{
+ args_t *args = (args_t*) usr;
+ int ret = args->tgt_is_bed ? regidx_parse_bed(line, chr_beg, chr_end, beg, end, NULL, NULL) : regidx_parse_tab(line, chr_beg, chr_end, beg, end, NULL, NULL);
+ if ( ret<0 ) return ret;
+ *((char **)payload) = strdup(line);
+ return 0;
+}
+void free_payload(void *payload)
+{
+ char *str = *((char**)payload);
+ free(str);
+}
+
void remove_id(args_t *args, bcf1_t *line, rm_tag_t *tag)
{
bcf_update_id(args->hdr,line,NULL);
}
void remove_filter(args_t *args, bcf1_t *line, rm_tag_t *tag)
{
+ if ( tag->key && tag->hdr_id<0 )
+ {
+ error("Error: Cannot proceed, not even with the --force option, bad things could happen.\n"
+ " Note that \"bcftools annotate -x FILTER\" can be used to remove ALL filters.\n"
+ " Even better, use \"bcftools view -h\" and \"bcftools reheader\" to fix the header!\n"
+ );
+ }
if ( !tag->key ) bcf_update_filter(args->hdr, line, NULL, args->flt_keep_pass);
else bcf_remove_filter(args->hdr, line, tag->hdr_id, args->flt_keep_pass);
}
memmove(&hdr->hrec[i],&hdr->hrec[i+1],(hdr->nhrec-i)*sizeof(bcf_hrec_t*));
bcf_hrec_destroy(hrec);
}
- if ( nrm ) bcf_hdr_sync(hdr);
+ if ( nrm ) {
+ if (bcf_hdr_sync(hdr) < 0)
+ error_errno("[%s] Failed to update header", __func__);
+ }
}
static void init_remove_annots(args_t *args)
tag->handler = remove_filter;
tag->key = strdup(str.s);
tag->hdr_id = bcf_hdr_id2int(args->hdr, BCF_DT_ID, tag->key);
- if ( !bcf_hdr_idinfo_exists(args->hdr,BCF_HL_FLT,tag->hdr_id) ) error("Cannot remove %s, not defined in the header.\n", str.s);
- if ( !args->keep_sites ) bcf_hdr_remove(args->hdr_out,BCF_HL_FLT,tag->key);
+ if ( !bcf_hdr_idinfo_exists(args->hdr,BCF_HL_FLT,tag->hdr_id) )
+ {
+ if ( args->keep_sites )
+ error("Error: The filter \"%s\" is not defined in the header, cannot use the -k option\n", str.s);
+ else
+ fprintf(bcftools_stderr,"Warning: The filter \"%s\" is not defined in the header\n", str.s);
+ }
+ else if ( !args->keep_sites ) bcf_hdr_remove(args->hdr_out,BCF_HL_FLT,tag->key);
}
else
{
int id = bcf_hdr_id2int(args->hdr,BCF_DT_ID,str.s);
if ( !bcf_hdr_idinfo_exists(args->hdr,type,id) )
{
- fprintf(bcftools_stderr,"Warning: The tag \"%s\" not defined in the header\n", str.s);
- args->nrm--;
+ if ( args->keep_sites )
+ error("Error: The tag \"%s\" is not defined in the header, cannot use the -k option\n", str.s);
+ else
+ fprintf(bcftools_stderr,"Warning: The tag \"%s\" not defined in the header\n", str.s);
+
+ tag->key = strdup(str.s);
+ if ( type==BCF_HL_INFO ) tag->handler = remove_info_tag;
+ else if ( type==BCF_HL_FMT ) tag->handler = remove_format_tag;
}
else if ( (type==BCF_HL_FMT && keep_fmt) || (type==BCF_HL_INFO && keep_info) )
{
}
khash_str2int_destroy_free(keep);
if ( !args->nrm ) error("No matching tag in -x %s\n", args->remove_annots);
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
}
static void init_header_lines(args_t *args)
{
if ( bcf_hdr_append(args->hdr_out,str.s) ) error("Could not parse %s: %s\n", args->header_fname, str.s);
bcf_hdr_append(args->hdr,str.s); // the input file may not have the header line if run with -h (and nothing else)
}
- hts_close(file);
+ if ( hts_close(file)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->header_fname);
free(str.s);
- bcf_hdr_sync(args->hdr_out);
- bcf_hdr_sync(args->hdr);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update output header", __func__);
+ if (bcf_hdr_sync(args->hdr) < 0)
+ error_errno("[%s] Failed to update input header", __func__);
}
static int setter_filter(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with FILTER (yet?)\n");
+
// note: so far this works only with one filter, not a list of filters
annot_line_t *tab = (annot_line_t*) data;
if ( tab->cols[col->icol] && tab->cols[col->icol][0]=='.' && !tab->cols[col->icol][1] ) return 0; // don't replace with "."
}
static int setter_id(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with ID (yet?)\n");
+
// possible cases:
// IN ANNOT OUT ACHIEVED_BY
// x y x -c +ID
}
static int setter_qual(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with QUAL (yet?)\n");
+
annot_line_t *tab = (annot_line_t*) data;
char *str = tab->cols[col->icol];
if ( str[0]=='.' && str[1]==0 ) return 0; // empty
line->qual = strtod(str, &str);
if ( str == tab->cols[col->icol] )
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
return 0;
}
static int vcf_setter_qual(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
}
static int setter_info_flag(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with INFO type=Flag (yet?)\n");
+
annot_line_t *tab = (annot_line_t*) data;
char *str = tab->cols[col->icol];
if ( str[0]=='.' && str[1]==0 ) return 0;
if ( str[0]=='1' && str[1]==0 ) return bcf_update_info_flag(args->hdr_out,line,col->hdr_key_dst,NULL,1);
if ( str[0]=='0' && str[1]==0 ) return bcf_update_info_flag(args->hdr_out,line,col->hdr_key_dst,NULL,0);
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
return -1;
}
static int vcf_setter_info_flag(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
static int setter_ARinfo_int32(args_t *args, bcf1_t *line, annot_col_t *col, int nals, char **als, int ntmpi)
{
if ( col->number==BCF_VL_A && ntmpi!=nals-1 && (ntmpi!=1 || args->tmpi[0]!=bcf_int32_missing || args->tmpi[1]!=bcf_int32_vector_end) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", ntmpi,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", ntmpi,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
else if ( col->number==BCF_VL_R && ntmpi!=nals && (ntmpi!=1 || args->tmpi[0]!=bcf_int32_missing || args->tmpi[1]!=bcf_int32_vector_end) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", ntmpi,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", ntmpi,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ndst = col->number==BCF_VL_A ? line->n_allele - 1 : line->n_allele;
int *map = vcmp_map_ARvalues(args->vcmp,ndst,nals,als,line->n_allele,line->d.allele);
- if ( !map ) error("REF alleles not compatible at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !map ) error("REF alleles not compatible at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
// fill in any missing values in the target VCF (or all, if not present)
int ntmpi2 = bcf_get_info_float(args->hdr, line, col->hdr_key_dst, &args->tmpi2, &args->mtmpi2);
static int setter_info_int(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
annot_line_t *tab = (annot_line_t*) data;
- char *str = tab->cols[col->icol], *end = str;
- if ( str[0]=='.' && str[1]==0 ) return 0;
- int ntmpi = 0;
- while ( *end )
+ if ( !tab )
+ {
+ if ( col->merge_method!=MM_SUM && col->merge_method!=MM_AVG && col->merge_method!=MM_MIN && col->merge_method!=MM_MAX && col->merge_method!=MM_APPEND )
+ error("Error: at the moment only the sum,avg,min,max,append options are supported with --merge-logic for INFO type=Integer\n");
+ }
+
+ int i,ntmpi = 0;
+ if ( tab )
{
- int val = strtol(str, &end, 10);
- if ( end==str )
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
- ntmpi++;
+ char *str = tab->cols[col->icol], *end = str;
+ if ( str[0]=='.' && str[1]==0 ) return 0;
+
+ while ( *end )
+ {
+ int val = strtol(str, &end, 10);
+ if ( end==str )
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
+ ntmpi++;
+ hts_expand(int32_t,ntmpi,args->mtmpi,args->tmpi);
+ args->tmpi[ntmpi-1] = val;
+ str = end+1;
+ }
+ if ( col->merge_method!=MM_FIRST )
+ {
+ if ( !col->mm_dbl_nused )
+ {
+ col->mm_dbl_nused = ntmpi;
+ hts_expand(double,col->mm_dbl_nused,col->mm_dbl_nalloc,col->mm_dbl);
+ for (i=0; i<ntmpi; i++)
+ col->mm_dbl[i] = args->tmpi[i];
+ }
+ else
+ {
+ if ( col->merge_method==MM_APPEND )
+ {
+ int nori = col->mm_dbl_nused;
+ col->mm_dbl_nused += ntmpi;
+ hts_expand(double,col->mm_dbl_nused,col->mm_dbl_nalloc,col->mm_dbl);
+ for (i=0; i<ntmpi; i++)
+ col->mm_dbl[i+nori] = args->tmpi[i];
+ }
+ else
+ {
+ if ( ntmpi!=col->mm_dbl_nused ) error("Error: cannot merge fields of unequal length\n");
+ if ( col->merge_method==MM_SUM || col->merge_method==MM_AVG )
+ for (i=0; i<ntmpi; i++) col->mm_dbl[i] += args->tmpi[i];
+ else if ( col->merge_method==MM_MIN )
+ for (i=0; i<ntmpi; i++) { if ( col->mm_dbl[i] > args->tmpi[i] ) col->mm_dbl[i] = args->tmpi[i]; }
+ else if ( col->merge_method==MM_MAX )
+ for (i=0; i<ntmpi; i++) { if ( col->mm_dbl[i] < args->tmpi[i] ) col->mm_dbl[i] = args->tmpi[i]; }
+ }
+ }
+ col->mm_dbl_ndat++;
+ }
+ }
+ else if ( col->merge_method==MM_SUM || col->merge_method==MM_MIN || col->merge_method==MM_MAX || col->merge_method==MM_APPEND )
+ {
+ ntmpi = col->mm_dbl_nused;
hts_expand(int32_t,ntmpi,args->mtmpi,args->tmpi);
- args->tmpi[ntmpi-1] = val;
- str = end+1;
+ for (i=0; i<ntmpi; i++) args->tmpi[i] = col->mm_dbl[i];
+ col->mm_dbl_nused = col->mm_dbl_ndat = 0;
+ }
+ else if ( col->merge_method==MM_AVG )
+ {
+ ntmpi = col->mm_dbl_nused;
+ hts_expand(int32_t,ntmpi,args->mtmpi,args->tmpi);
+ for (i=0; i<ntmpi; i++) args->tmpi[i] = col->mm_dbl[i]/col->mm_dbl_ndat;
+ col->mm_dbl_nused = col->mm_dbl_ndat = 0;
}
if ( col->number==BCF_VL_A || col->number==BCF_VL_R )
static int setter_ARinfo_real(args_t *args, bcf1_t *line, annot_col_t *col, int nals, char **als, int ntmpf)
{
if ( col->number==BCF_VL_A && ntmpf!=nals-1 && (ntmpf!=1 || !bcf_float_is_missing(args->tmpf[0]) || !bcf_float_is_vector_end(args->tmpf[0])) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", ntmpf,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", ntmpf,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
else if ( col->number==BCF_VL_R && ntmpf!=nals && (ntmpf!=1 || !bcf_float_is_missing(args->tmpf[0]) || !bcf_float_is_vector_end(args->tmpf[0])) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", ntmpf,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", ntmpf,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ndst = col->number==BCF_VL_A ? line->n_allele - 1 : line->n_allele;
int *map = vcmp_map_ARvalues(args->vcmp,ndst,nals,als,line->n_allele,line->d.allele);
- if ( !map ) error("REF alleles not compatible at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !map ) error("REF alleles not compatible at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
// fill in any missing values in the target VCF (or all, if not present)
int ntmpf2 = bcf_get_info_float(args->hdr, line, col->hdr_key_dst, &args->tmpf2, &args->mtmpf2);
static int setter_info_real(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
annot_line_t *tab = (annot_line_t*) data;
- char *str = tab->cols[col->icol], *end = str;
- if ( str[0]=='.' && str[1]==0 ) return 0;
- int ntmpf = 0;
- while ( *end )
+ if ( !tab )
+ {
+ if ( col->merge_method!=MM_SUM && col->merge_method!=MM_AVG && col->merge_method!=MM_MIN && col->merge_method!=MM_MAX && col->merge_method!=MM_APPEND )
+ error("Error: at the moment only the sum,avg,min,max,append options are supported with --merge-logic for INFO type=Float\n");
+ }
+
+ int i,ntmpf = 0;
+ if ( tab )
+ {
+ char *str = tab->cols[col->icol], *end = str;
+ if ( str[0]=='.' && str[1]==0 ) return 0;
+
+ while ( *end )
+ {
+ double val = strtod(str, &end);
+ if ( end==str )
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
+ ntmpf++;
+ hts_expand(float,ntmpf,args->mtmpf,args->tmpf);
+ args->tmpf[ntmpf-1] = val;
+ str = end+1;
+ }
+ if ( col->merge_method!=MM_FIRST )
+ {
+ if ( !col->mm_dbl_nused )
+ {
+ col->mm_dbl_nused = ntmpf;
+ hts_expand(double,col->mm_dbl_nused,col->mm_dbl_nalloc,col->mm_dbl);
+ for (i=0; i<ntmpf; i++)
+ col->mm_dbl[i] = args->tmpf[i];
+ }
+ else
+ {
+ if ( col->merge_method==MM_APPEND )
+ {
+ int nori = col->mm_dbl_nused;
+ col->mm_dbl_nused += ntmpf;
+ hts_expand(double,col->mm_dbl_nused,col->mm_dbl_nalloc,col->mm_dbl);
+ for (i=0; i<ntmpf; i++)
+ col->mm_dbl[i+nori] = args->tmpf[i];
+ }
+ else
+ {
+ if ( ntmpf!=col->mm_dbl_nused ) error("Error: cannot merge fields of unequal length\n");
+ if ( col->merge_method==MM_SUM || col->merge_method==MM_AVG )
+ for (i=0; i<ntmpf; i++) col->mm_dbl[i] += args->tmpf[i];
+ else if ( col->merge_method==MM_MIN )
+ for (i=0; i<ntmpf; i++) { if ( col->mm_dbl[i] > args->tmpf[i] ) col->mm_dbl[i] = args->tmpf[i]; }
+ else if ( col->merge_method==MM_MAX )
+ for (i=0; i<ntmpf; i++) { if ( col->mm_dbl[i] < args->tmpf[i] ) col->mm_dbl[i] = args->tmpf[i]; }
+ }
+ }
+ col->mm_dbl_ndat++;
+ }
+ }
+ else if ( col->merge_method==MM_SUM || col->merge_method==MM_MIN || col->merge_method==MM_MAX || col->merge_method==MM_APPEND )
{
- double val = strtod(str, &end);
- if ( end==str )
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
- ntmpf++;
- hts_expand(float,ntmpf,args->mtmpf,args->tmpf);
- args->tmpf[ntmpf-1] = val;
- str = end+1;
+ ntmpf = col->mm_dbl_nused;
+ hts_expand(int32_t,ntmpf,args->mtmpf,args->tmpf);
+ for (i=0; i<ntmpf; i++) args->tmpf[i] = col->mm_dbl[i];
+ col->mm_dbl_nused = col->mm_dbl_ndat = 0;
+ }
+ else if ( col->merge_method==MM_AVG )
+ {
+ ntmpf = col->mm_dbl_nused;
+ hts_expand(int32_t,ntmpf,args->mtmpf,args->tmpf);
+ for (i=0; i<ntmpf; i++) args->tmpf[i] = col->mm_dbl[i]/col->mm_dbl_ndat;
+ col->mm_dbl_nused = col->mm_dbl_ndat = 0;
}
if ( col->number==BCF_VL_A || col->number==BCF_VL_R )
int copy_string_field(char *src, int isrc, int src_len, kstring_t *dst, int idst); // see vcfmerge.c
static int setter_ARinfo_string(args_t *args, bcf1_t *line, annot_col_t *col, int nals, char **als)
{
+ assert( col->merge_method==MM_FIRST );
+
int nsrc = 1, lsrc = 0;
while ( args->tmps[lsrc] )
{
lsrc++;
}
if ( col->number==BCF_VL_A && nsrc!=nals-1 && (nsrc!=1 || args->tmps[0]!='.' || args->tmps[1]!=0 ) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", nsrc,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", nsrc,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
else if ( col->number==BCF_VL_R && nsrc!=nals && (nsrc!=1 || args->tmps[0]!='.' || args->tmps[1]!=0 ) )
- error("Incorrect number of values (%d) for the %s tag at %s:%d\n", nsrc,col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values (%d) for the %s tag at %s:%"PRId64"\n", nsrc,col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ndst = col->number==BCF_VL_A ? line->n_allele - 1 : line->n_allele;
int *map = vcmp_map_ARvalues(args->vcmp,ndst,nals,als,line->n_allele,line->d.allele);
- if ( !map ) error("REF alleles not compatible at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !map ) error("REF alleles not compatible at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
// fill in any missing values in the target VCF (or all, if not present)
int i, empty = 0, nstr, mstr = args->tmpks.m;
bcf_update_info_string(args->hdr_out,line,col->hdr_key_dst,args->tmpks.s);
return 0;
}
+void khash_str2int_clear_free(void *_hash)
+{
+ khash_t(str2int) *hash = (khash_t(str2int)*)_hash;
+ khint_t k;
+ if (hash == 0) return;
+ for (k = 0; k < kh_end(hash); ++k)
+ if (kh_exist(hash, k)) free((char*)kh_key(hash, k));
+ kh_clear(str2int, hash);
+}
static int setter_info_str(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( col->replace==REPLACE_MISSING && col->number!=BCF_VL_A && col->number!=BCF_VL_R )
+ {
+ int ret = bcf_get_info_string(args->hdr, line, col->hdr_key_dst, &args->tmps2, &args->mtmps2);
+ if ( ret>0 && (args->tmps2[0]!='.' || args->tmps2[1]!=0) ) return 0;
+ }
+
annot_line_t *tab = (annot_line_t*) data;
- int len = strlen(tab->cols[col->icol]);
- if ( !len ) return 0;
- hts_expand(char,len+1,args->mtmps,args->tmps);
- memcpy(args->tmps,tab->cols[col->icol],len+1);
- if ( args->tmps[0]=='.' && args->tmps[1]==0 ) return 0;
+
+ int len = 0;
+ if ( tab )
+ {
+ len = strlen(tab->cols[col->icol]);
+ if ( !len ) return 0;
+ if ( len==1 && tab->cols[col->icol][0]=='.' ) return 0;
+ }
- if ( col->number==BCF_VL_A || col->number==BCF_VL_R )
- return setter_ARinfo_string(args,line,col,tab->nals,tab->als);
+ if ( col->merge_method!=MM_FIRST )
+ {
+ if ( col->number==BCF_VL_A || col->number==BCF_VL_R )
+ error("Error: the --merge-logic option cannot be used with INFO tags Type=String,Number={A,R,G}\n");
- if ( col->replace==REPLACE_MISSING )
+ if ( data )
+ {
+ assert( col->merge_method==MM_APPEND || col->merge_method==MM_UNIQUE );
+ if ( col->merge_method==MM_UNIQUE )
+ {
+ if ( !col->mm_str_hash ) col->mm_str_hash = (khash_t(str2int)*)khash_str2int_init();
+ if ( khash_str2int_has_key(col->mm_str_hash, tab->cols[col->icol]) ) return 0;
+ khash_str2int_inc(col->mm_str_hash, strdup(tab->cols[col->icol]));
+ }
+
+ if ( col->mm_kstr.l ) kputc(',',&col->mm_kstr);
+ kputs(tab->cols[col->icol], &col->mm_kstr);
+ return 0;
+ }
+
+ if ( col->mm_kstr.l )
+ {
+ hts_expand(char,col->mm_kstr.l+1,args->mtmps,args->tmps);
+ memcpy(args->tmps,col->mm_kstr.s,col->mm_kstr.l+1);
+ }
+ else
+ return 0;
+
+ if ( !data ) // flush the line
+ {
+ if ( col->merge_method==MM_UNIQUE )
+ khash_str2int_clear_free(col->mm_str_hash);
+ col->mm_kstr.l = 0;
+ }
+ }
+ else
{
- int ret = bcf_get_info_string(args->hdr, line, col->hdr_key_dst, &args->tmps2, &args->mtmps2);
- if ( ret>0 && (args->tmps2[0]!='.' || args->tmps2[1]!=0) ) return 0;
+ assert(tab);
+ hts_expand(char,len+1,args->mtmps,args->tmps);
+ memcpy(args->tmps,tab->cols[col->icol],len+1);
+
+ if ( col->number==BCF_VL_A || col->number==BCF_VL_R )
+ return setter_ARinfo_string(args,line,col,tab->nals,tab->als);
}
bcf_update_info_string(args->hdr_out,line,col->hdr_key_dst,args->tmps);
bcf_update_info_string(args->hdr_out,line,col->hdr_key_dst,args->tmps);
return 0;
}
+static int genotypes_to_string(args_t *args, int nsrc1, int32_t *src, int nsmpl_dst, kstring_t *str)
+{
+ int i, isrc, idst;
+ int blen = nsrc1 > 1 ? nsrc1 + 1 : 1; // typically the genotypes take three bytes 0/1, no 0-termination is needed
+
+gt_length_too_big:
+ str->l = 0;
+ for (idst=0; idst<nsmpl_dst; idst++)
+ {
+ isrc = args->sample_map ? args->sample_map[idst] : idst;
+ if ( isrc==-1 )
+ {
+ kputc_('.', str);
+ for (i=1; i < blen; i++) kputc_(0, str);
+ continue;
+ }
+
+ size_t plen = str->l;
+ int32_t *ptr = src + isrc*nsrc1;
+ for (i=0; i<nsrc1 && ptr[i]!=bcf_int32_vector_end; i++)
+ {
+ if ( i ) kputc("/|"[bcf_gt_is_phased(ptr[i])], str);
+ if ( bcf_gt_is_missing(ptr[i]) ) kputc('.', str);
+ else kputw(bcf_gt_allele(ptr[i]), str);
+ }
+ if ( i==0 ) kputc('.', str);
+ if ( str->l - plen > blen )
+ {
+ // too many alternate alleles or ploidy is too large, the genotype does not fit
+ // three characters ("0/0" vs "10/10").
+ blen *= 2;
+ goto gt_length_too_big;
+ }
+ plen = str->l - plen;
+ while ( plen < blen )
+ {
+ kputc_(0, str);
+ plen++;
+ }
+ }
+ return 0;
+}
static int vcf_setter_format_gt(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
bcf1_t *rec = (bcf1_t*) data;
if ( nsrc==-3 ) return 0; // the tag is not present
if ( nsrc<=0 ) return 1; // error
+ // Genotypes are internally represented as integers. This is a complication when
+ // adding as a different Type=String field, such as FMT/newGT:=GT
+ if ( strcmp(col->hdr_key_src,col->hdr_key_dst) )
+ {
+ int nsmpl_dst = bcf_hdr_nsamples(args->hdr_out);
+ int nsmpl_src = bcf_hdr_nsamples(args->files->readers[1].header);
+ genotypes_to_string(args,nsrc/nsmpl_src,args->tmpi,nsmpl_dst,&args->tmpks);
+ return bcf_update_format_char(args->hdr_out,line,col->hdr_key_dst,args->tmpks.s,args->tmpks.l);
+ }
+
if ( !args->sample_map )
return bcf_update_genotypes(args->hdr_out,line,args->tmpi,nsrc);
}
static int setter_format_int(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with FORMAT tags (yet?)\n");
+
annot_line_t *tab = (annot_line_t*) data;
if ( col->icol+args->nsmpl_annot > tab->ncols )
- error("Incorrect number of values for %s at %s:%d\n",col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values for %s at %s:%"PRId64"\n",col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int nvals = count_vals(tab,col->icol,col->icol+args->nsmpl_annot);
hts_expand(int32_t,nvals*args->nsmpl_annot,args->mtmpi,args->tmpi);
char *end = str;
ptr[ival] = strtol(str, &end, 10);
if ( end==str )
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
ival++;
str = *end ? end+1 : end;
}
static int setter_format_real(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with FORMAT tags (yet?)\n");
+
annot_line_t *tab = (annot_line_t*) data;
if ( col->icol+args->nsmpl_annot > tab->ncols )
- error("Incorrect number of values for %s at %s:%d\n",col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values for %s at %s:%"PRId64"\n",col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int nvals = count_vals(tab,col->icol,col->icol+args->nsmpl_annot);
hts_expand(float,nvals*args->nsmpl_annot,args->mtmpf,args->tmpf);
char *end = str;
ptr[ival] = strtod(str, &end);
if ( end==str )
- error("Could not parse %s at %s:%d .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1,tab->cols[col->icol]);
+ error("Could not parse %s at %s:%"PRId64" .. [%s]\n", col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1,tab->cols[col->icol]);
ival++;
str = *end ? end+1 : end;
}
static int setter_format_str(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
+ if ( !data ) error("Error: the --merge-logic option cannot be used with FORMAT tags (yet?)\n");
+
annot_line_t *tab = (annot_line_t*) data;
if ( col->icol+args->nsmpl_annot > tab->ncols )
- error("Incorrect number of values for %s at %s:%d\n",col->hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("Incorrect number of values for %s at %s:%"PRId64"\n",col->hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ismpl;
for (ismpl=0; ismpl<args->nsmpl_annot; ismpl++)
// create mapping from src to dst genotypes, haploid and diploid version
int nmap_hap = col->number==BCF_VL_G || col->number==BCF_VL_R ? rec->n_allele : rec->n_allele - 1;
int *map_hap = vcmp_map_ARvalues(args->vcmp,nmap_hap,line->n_allele,line->d.allele,rec->n_allele,rec->d.allele);
- if ( !map_hap ) error("REF alleles not compatible at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !map_hap ) error("REF alleles not compatible at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int i, j;
if ( rec->n_allele==line->n_allele )
}
int pld_src = determine_ploidy(rec->n_allele, args->tmpi, nsrc1, args->src_smpl_pld, nsmpl_src);
if ( pld_src<0 )
- error("Unexpected number of %s values (%d) for %d alleles at %s:%d\n", col->hdr_key_src,-pld_src, rec->n_allele, bcf_seqname(bcf_sr_get_header(args->files,1),rec),rec->pos+1);
+ error("Unexpected number of %s values (%d) for %d alleles at %s:%"PRId64"\n", col->hdr_key_src,-pld_src, rec->n_allele, bcf_seqname(bcf_sr_get_header(args->files,1),rec),(int64_t) rec->pos+1);
int pld_dst = determine_ploidy(line->n_allele, args->tmpi2, ndst1, args->dst_smpl_pld, nsmpl_dst);
if ( pld_dst<0 )
- error("Unexpected number of %s values (%d) for %d alleles at %s:%d\n", col->hdr_key_src,-pld_dst, line->n_allele, bcf_seqname(args->hdr,line),line->pos+1);
+ error("Unexpected number of %s values (%d) for %d alleles at %s:%"PRId64"\n", col->hdr_key_src,-pld_dst, line->n_allele, bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ndst1_new = pld_dst==1 ? line->n_allele : line->n_allele*(line->n_allele+1)/2;
if ( ndst1_new != ndst1 )
{
- if ( ndst1 ) error("todo: %s ndst1!=ndst .. %d %d at %s:%d\n",col->hdr_key_src,ndst1_new,ndst1,bcf_seqname(args->hdr,line),line->pos+1);
+ if ( ndst1 ) error("todo: %s ndst1!=ndst .. %d %d at %s:%"PRId64"\n",col->hdr_key_src,ndst1_new,ndst1,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
ndst1 = ndst1_new;
hts_expand(int32_t, ndst1*nsmpl_dst, args->mtmpi2, args->tmpi2);
}
if ( col->number==BCF_VL_G )
{
if ( args->src_smpl_pld[ii] > 0 && args->dst_smpl_pld[i] > 0 && args->src_smpl_pld[ii]!=args->dst_smpl_pld[i] )
- error("Sample ploidy differs at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ error("Sample ploidy differs at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
if ( !args->dst_smpl_pld[i] )
for (j=0; j<ndst1; j++) ptr_dst[j] = bcf_int32_missing;
}
}
static int vcf_setter_format_real(args_t *args, bcf1_t *line, annot_col_t *col, void *data)
{
-
bcf1_t *rec = (bcf1_t*) data;
int nsrc = bcf_get_format_float(args->files->readers[1].header,rec,col->hdr_key_src,&args->tmpf,&args->mtmpf);
if ( nsrc==-3 ) return 0; // the tag is not present
// create mapping from src to dst genotypes, haploid and diploid version
int nmap_hap = col->number==BCF_VL_G || col->number==BCF_VL_R ? rec->n_allele : rec->n_allele - 1;
int *map_hap = vcmp_map_ARvalues(args->vcmp,nmap_hap,line->n_allele,line->d.allele,rec->n_allele,rec->d.allele);
- if ( !map_hap ) error("REF alleles not compatible at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !map_hap ) error("REF alleles not compatible at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int i, j;
if ( rec->n_allele==line->n_allele )
}
int pld_src = determine_ploidy(rec->n_allele, args->tmpi, nsrc1, args->src_smpl_pld, nsmpl_src);
if ( pld_src<0 )
- error("Unexpected number of %s values (%d) for %d alleles at %s:%d\n", col->hdr_key_src,-pld_src, rec->n_allele, bcf_seqname(bcf_sr_get_header(args->files,1),rec),rec->pos+1);
+ error("Unexpected number of %s values (%d) for %d alleles at %s:%"PRId64"\n", col->hdr_key_src,-pld_src, rec->n_allele, bcf_seqname(bcf_sr_get_header(args->files,1),rec),(int64_t) rec->pos+1);
int pld_dst = determine_ploidy(line->n_allele, args->tmpi2, ndst1, args->dst_smpl_pld, nsmpl_dst);
if ( pld_dst<0 )
- error("Unexpected number of %s values (%d) for %d alleles at %s:%d\n", col->hdr_key_src,-pld_dst, line->n_allele, bcf_seqname(args->hdr,line),line->pos+1);
+ error("Unexpected number of %s values (%d) for %d alleles at %s:%"PRId64"\n", col->hdr_key_src,-pld_dst, line->n_allele, bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
int ndst1_new = pld_dst==1 ? line->n_allele : line->n_allele*(line->n_allele+1)/2;
if ( ndst1_new != ndst1 )
{
- if ( ndst1 ) error("todo: %s ndst1!=ndst .. %d %d at %s:%d\n",col->hdr_key_src,ndst1_new,ndst1,bcf_seqname(args->hdr,line),line->pos+1);
+ if ( ndst1 ) error("todo: %s ndst1!=ndst .. %d %d at %s:%"PRId64"\n",col->hdr_key_src,ndst1_new,ndst1,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
ndst1 = ndst1_new;
hts_expand(float, ndst1*nsmpl_dst, args->mtmpf2, args->tmpf2);
}
if ( col->number==BCF_VL_G )
{
if ( args->src_smpl_pld[ii] > 0 && args->dst_smpl_pld[i] > 0 && args->src_smpl_pld[ii]!=args->dst_smpl_pld[i] )
- error("Sample ploidy differs at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ error("Sample ploidy differs at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
if ( !args->dst_smpl_pld[i] )
for (j=0; j<ndst1; j++) bcf_float_set_missing(ptr_dst[j]);
}
args->tmps = args->tmpp[0]; // tmps might be realloced
if ( ret==-3 ) return 0; // the tag is not present
if ( ret<=0 ) return 1; // error
- return core_setter_format_str(args,line,col,args->tmpp);
+ if ( strcmp("GT",col->hdr_key_dst) )
+ return core_setter_format_str(args,line,col,args->tmpp);
+
+ // Genotypes are internally represented as integers. This is a complication for FMT/GT:=oldGT
+ // First determine the maximum number of alleles per-sample ndst1
+ int nsmpl_src = bcf_hdr_nsamples(args->files->readers[1].header);
+ int nsmpl_dst = bcf_hdr_nsamples(args->hdr_out);
+ int isrc,idst, ndst1 = 0, nsrc1 = ret / nsmpl_src;
+ char *ptr = args->tmps, *ptr_end = ptr + ret;
+ while ( ptr < ptr_end )
+ {
+ char *smpl_end = ptr + nsrc1;
+ int n = 1;
+ while ( ptr < smpl_end )
+ {
+ if ( *ptr=='/' || *ptr=='|' ) n++;
+ ptr++;
+ }
+ if ( ndst1 < n ) ndst1 = n;
+ }
+ assert( ndst1 );
+
+ int ndst = ndst1*nsmpl_dst;
+ hts_expand(int32_t,ndst,args->mtmpi,args->tmpi);
+ hts_expand(char,ret+1,args->mtmps,args->tmps); args->tmps[ret] = 0; // the FORMAT string may not be 0-terminated
+ for (idst=0; idst<nsmpl_dst; idst++)
+ {
+ int i = 0, is_phased = 0;
+ int32_t *dst = args->tmpi + idst*ndst1;
+ isrc = args->sample_map ? args->sample_map[idst] : idst;
+ if ( isrc==-1 )
+ {
+ dst[0] = bcf_gt_missing;
+ for (i=1; i<ndst1; i++) dst[i] = bcf_int32_vector_end;
+ continue;
+ }
+ char *beg = args->tmps + isrc*nsrc1, *tmp;
+ char *keep_ptr = beg+nsrc1, keep = *keep_ptr; *keep_ptr = 0;
+ while ( *beg )
+ {
+ char *end = beg;
+ while ( *end && *end!='/' && *end!='|' ) end++;
+ if ( *beg=='.' && end-beg==1 ) dst[i] = bcf_gt_missing;
+ else
+ {
+ if ( *end=='|' ) is_phased = 1;
+ dst[i] = strtol(beg, &tmp, 10);
+ if ( tmp!=end )
+ error("Could not parse the %s field at %s:%"PRId64" in %s\n", col->hdr_key_src,bcf_seqname(args->files->readers[1].header,rec),(int64_t) rec->pos+1,args->targets_fname);
+ if ( dst[i] >= line->n_allele )
+ error("The source allele index is bigger than the number of destination alleles at %s:%"PRId64"\n", bcf_seqname(args->files->readers[1].header,rec),(int64_t) rec->pos+1);
+ dst[i] = is_phased ? bcf_gt_phased(dst[i]) : bcf_gt_unphased(dst[i]);
+ }
+ beg = *end ? end+1 : end;
+ i++;
+ }
+ *keep_ptr = keep;
+ for (; i<ndst1; i++) dst[i] = bcf_int32_vector_end;
+ }
+ return bcf_update_genotypes(args->hdr_out,line,args->tmpi,ndst);
}
static int init_sample_map(args_t *args, bcf_hdr_t *src, bcf_hdr_t *dst)
{
args->sample_map = (int*) malloc(sizeof(int)*args->nsample_map);
for (i=0; i<args->nsample_map; i++) args->sample_map[i] = -1;
- // possible todo: could do with smpl_ilist only
- smpl_ilist_t *ilist = smpl_ilist_init(dst, args->sample_names, args->sample_is_file, SMPL_STRICT);
- if ( !ilist || !ilist->n ) error("Could not parse: %s\n", args->sample_names);
- char **samples = (char**) malloc(sizeof(char*)*ilist->n);
- for (i=0; i<ilist->n; i++) samples[i] = strdup(dst->samples[i]);
+ int flags = !src ? SMPL_STRICT|SMPL_SINGLE : SMPL_STRICT|SMPL_SINGLE|SMPL_PAIR2; // is vcf vs tab annotation file
+ smpl_ilist_t *ilist = smpl_ilist_init(dst, args->sample_names, args->sample_is_file, flags); // gives mapping dst->src
+ if ( !ilist || !ilist->n ) error("Could not parse the samples: %s\n", args->sample_names);
args->nsmpl_annot = ilist->n;
- smpl_ilist_destroy(ilist);
int need_sample_map = args->nsmpl_annot==bcf_hdr_nsamples(dst) ? 0 : 1;
- if ( !src )
- {
- // tab annotation file
- for (i=0; i<args->nsmpl_annot; i++)
- {
- int idst = bcf_hdr_id2int(dst, BCF_DT_SAMPLE, samples[i]);
- if ( idst==-1 ) error("Sample \"%s\" not found in the destination file\n", samples[i]);
- args->sample_map[idst] = i;
- if ( idst!=i ) need_sample_map = 1;
- }
- }
- else
+ for (i=0; i<args->nsmpl_annot; i++)
{
- // vcf annotation file
- for (i=0; i<args->nsmpl_annot; i++)
+ int idst = ilist->idx[i];
+ const char *src_name = ilist->pair && ilist->pair[i] ? ilist->pair[i] : bcf_hdr_int2id(dst, BCF_DT_SAMPLE, idst);
+ int isrc = i;
+ if ( src ) // the annotation file is a VCF, not a tab-delimited file
{
- int isrc, idst;
- char *ss = samples[i], *se = samples[i];
- while ( *se && !isspace(*se) ) se++;
- if ( !*se )
- {
- // only one sample name
- isrc = bcf_hdr_id2int(src, BCF_DT_SAMPLE,ss);
- if ( isrc==-1 ) error("Sample \"%s\" not found in the source file\n", ss);
- idst = bcf_hdr_id2int(dst, BCF_DT_SAMPLE,ss);
- if ( idst==-1 ) error("Sample \"%s\" not found in the destination file\n", ss);
- args->sample_map[idst] = isrc;
- if ( idst!=isrc ) need_sample_map = 1;
- continue;
- }
- *se = 0;
- isrc = bcf_hdr_id2int(src, BCF_DT_SAMPLE,ss);
- if ( isrc==-1 ) error("Sample \"%s\" not found in the source file\n", ss);
-
- ss = se+1;
- while ( isspace(*ss) ) ss++;
- se = ss;
- while ( *se && !isspace(*se) ) se++;
-
- idst = bcf_hdr_id2int(dst, BCF_DT_SAMPLE,ss);
- if ( idst==-1 ) error("Sample \"%s\" not found in the destination file\n", ss);
-
- args->sample_map[idst] = isrc;
- if ( idst!=isrc ) need_sample_map = 1;
+ isrc = bcf_hdr_id2int(src, BCF_DT_SAMPLE, src_name);
+ if ( isrc==-1 ) error("Sample \"%s\" not found in the annotation file\n", src_name);
}
+ if ( isrc!=idst ) need_sample_map = 1;
+ args->sample_map[idst] = isrc;
}
- for (i=0; i<args->nsmpl_annot; i++) free(samples[i]);
- free(samples);
+ smpl_ilist_destroy(ilist);
return need_sample_map;
}
static char *columns_complement(char *columns, void **skip_info, void **skip_fmt)
kputsn(ss, se-ss, &str);
if ( !str.s[0] || !strcasecmp("-",str.s) ) ;
else if ( !strcasecmp("CHROM",str.s) ) args->chr_idx = icol;
- else if ( !strcasecmp("POS",str.s) ) args->from_idx = icol;
- else if ( !strcasecmp("FROM",str.s) ) args->from_idx = icol;
- else if ( !strcasecmp("TO",str.s) ) args->to_idx = icol;
+ else if ( !strcasecmp("POS",str.s) ) args->beg_idx = icol;
+ else if ( !strcasecmp("FROM",str.s) || !strcasecmp("BEG",str.s) ) args->beg_idx = icol;
+ else if ( !strcasecmp("TO",str.s) || !strcasecmp("END",str.s) ) args->end_idx = icol;
else if ( !strcasecmp("REF",str.s) )
{
if ( args->tgts_is_vcf )
bcf_hrec_format(hrec, &tmp);
bcf_hdr_append(args->hdr_out, tmp.s);
}
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
}
}
else if ( !strcasecmp("QUAL",str.s) )
tmp.l = 0;
bcf_hrec_format(hrec, &tmp);
bcf_hdr_append(args->hdr_out, tmp.s);
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
int hdr_id = bcf_hdr_id2int(args->hdr_out, BCF_DT_ID, hrec->vals[k]);
args->ncols++; args->cols = (annot_col_t*) realloc(args->cols,sizeof(annot_col_t)*args->ncols);
annot_col_t *col = &args->cols[args->ncols-1];
tmp.l = 0;
bcf_hrec_format(hrec, &tmp);
bcf_hdr_append(args->hdr_out, tmp.s);
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
int hdr_id = bcf_hdr_id2int(args->hdr_out, BCF_DT_ID, hrec->vals[k]);
args->ncols++; args->cols = (annot_col_t*) realloc(args->cols,sizeof(annot_col_t)*args->ncols);
annot_col_t *col = &args->cols[args->ncols-1];
tmp.l = 0;
bcf_hrec_format_rename(hrec, key_dst, &tmp);
bcf_hdr_append(args->hdr_out, tmp.s);
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
}
int hdr_id = bcf_hdr_id2int(args->hdr_out, BCF_DT_ID, key_dst);
if ( !bcf_hdr_idinfo_exists(args->hdr_out,BCF_HL_FMT,hdr_id) )
{
if ( replace==REPLACE_NON_MISSING ) error("Apologies, the -INFO/TAG feature has not been implemented yet.\n");
if ( replace==SET_OR_APPEND ) error("Apologies, the =INFO/TAG feature has not been implemented yet.\n");
- char *key_dst = !strncasecmp("INFO/",str.s,5) ? str.s + 5 : str.s;
+ int explicit_info = 0;
+ char *key_dst;
+ if ( !strncasecmp("INFO/",str.s,5) )
+ {
+ key_dst = str.s + 5;
+ explicit_info = 1;
+ }
+ else
+ key_dst = str.s;
char *key_src = strstr(key_dst,":=");
if ( key_src )
{
*key_src = 0;
key_src += 2;
- if ( !strncasecmp("INFO/",key_src,5) ) key_src += 5;
+ if ( !strncasecmp("INFO/",key_src,5) )
+ {
+ key_src += 5;
+ explicit_info = 1;
+ }
+ else if ( !strncasecmp("FMT/",key_src,4) || !strncasecmp("FORMAT/",key_src,5) )
+ {
+ key_src[-2] = ':';
+ error("Did you mean \"FMT/%s\" rather than \"%s\"?\n",str.s,str.s);
+ }
}
else
key_src = key_dst;
if ( args->tgts_is_vcf ) // reading annotations from a VCF, add a new header line
{
bcf_hrec_t *hrec = bcf_hdr_get_hrec(args->files->readers[1].header, BCF_HL_INFO, "ID", key_src, NULL);
- if ( !hrec ) error("The tag \"%s\" is not defined in %s\n", str.s,args->files->readers[1].fname);
+ if ( !hrec )
+ {
+ if ( !explicit_info && bcf_hdr_get_hrec(args->files->readers[1].header, BCF_HL_FMT, "ID", key_src, NULL) )
+ error("Did you mean \"FMT/%s\" rather than \"%s\"?\n",str.s,str.s);
+ fprintf(bcftools_stderr,"[%s] %d\n",key_src,explicit_info);
+ error("The tag \"%s\" is not defined in %s\n", key_src,args->files->readers[1].fname);
+ }
tmp.l = 0;
bcf_hrec_format_rename(hrec, key_dst, &tmp);
bcf_hdr_append(args->hdr_out, tmp.s);
- bcf_hdr_sync(args->hdr_out);
+ if (bcf_hdr_sync(args->hdr_out) < 0)
+ error_errno("[%s] Failed to update header", __func__);
hdr_id = bcf_hdr_id2int(args->hdr_out, BCF_DT_ID, key_dst);
}
else
}
free(str.s);
free(tmp.s);
- if ( args->to_idx==-1 ) args->to_idx = args->from_idx;
free(args->columns);
if ( skip_info ) khash_str2int_destroy_free(skip_info);
if ( skip_fmt ) khash_str2int_destroy_free(skip_fmt);
else if ( sample_map_ok<0 )
error("No matching samples in source and destination file?\n");
}
+static void init_merge_method(args_t *args)
+{
+ int i;
+ for (i=0; i<args->ncols; i++)
+ {
+ args->cols[i].merge_method = MM_FIRST;
+ args->cols[i].mm_str_hash = NULL;
+ args->cols[i].mm_dbl = NULL;
+ args->cols[i].mm_dbl_nalloc = args->cols[i].mm_dbl_nused = args->cols[i].mm_dbl_ndat = 0;
+ memset(&args->cols[i].mm_kstr, 0, sizeof(args->cols[i].mm_kstr));
+ }
+ if ( !args->merge_method_str ) return;
+ if ( args->tgts_is_vcf ) error("Error: the --merge-logic is intended for use with BED or TAB-delimited files only.\n");
+ if ( !args->tgt_idx ) error("Error: BEG,END (or FROM,TO) columns are expected with the --merge-logic option.\n");
+ char *sb = args->merge_method_str;
+ while ( *sb )
+ {
+ char *se = sb;
+ while ( *se && *se!=',' ) se++;
+ args->tmpks.l = 0;
+ kputsn(sb, se-sb, &args->tmpks);
+ kputc(0, &args->tmpks);
+ char *mm_type_str = args->tmpks.s + args->tmpks.l;
+ while ( *mm_type_str!=':' && mm_type_str > args->tmpks.s ) mm_type_str--;
+ if ( *mm_type_str!=':' )
+ error("Error: could not parse the argument to --merge-logic: %s\n", args->merge_method_str);
+ *mm_type_str = 0;
+ mm_type_str++;
+ int mm_type = MM_FIRST;
+ if ( !strcasecmp("unique",mm_type_str) ) mm_type = MM_UNIQUE;
+ else if ( !strcasecmp("append",mm_type_str) ) mm_type = MM_APPEND;
+ else if ( !strcasecmp("sum",mm_type_str) ) mm_type = MM_SUM;
+ else if ( !strcasecmp("avg",mm_type_str) ) mm_type = MM_AVG;
+ else if ( !strcasecmp("min",mm_type_str) ) mm_type = MM_MIN;
+ else if ( !strcasecmp("max",mm_type_str) ) mm_type = MM_MAX;
+ else error("Error: could not parse --merge-logic %s, the logic \"%s\" is not recognised\n", args->merge_method_str,mm_type_str);
+ for (i=0; i<args->ncols; i++)
+ {
+ if ( strcmp(args->cols[i].hdr_key_dst,args->tmpks.s) ) continue;
+ if ( mm_type==MM_APPEND && args->cols[i].number!=BCF_VL_VAR )
+ error("Error: --merge-logic append can be requested only for tags of variable length (Number=.)\n");
+ args->cols[i].merge_method = mm_type;
+ break;
+ }
+ if ( i==args->ncols ) error("No such tag in the destination file: %s\n", args->tmpks.s);
+ sb = *se ? se + 1 : se;
+ }
+}
static void rename_chrs(args_t *args, char *fname)
{
{
if ( !args->columns ) error("The -c option not given\n");
if ( args->chr_idx==-1 ) error("The -c CHROM option not given\n");
- if ( args->from_idx==-1 ) error("The -c POS option not given\n");
- if ( args->to_idx==-1 ) args->to_idx = -args->from_idx - 1;
-
- args->tgts = bcf_sr_regions_init(args->targets_fname,1,args->chr_idx,args->from_idx,args->to_idx);
- if ( !args->tgts ) error("Could not initialize the annotation file: %s\n", args->targets_fname);
- if ( !args->tgts->tbx ) error("Expected tabix-indexed annotation file: %s\n", args->targets_fname);
+ if ( args->beg_idx==-1 ) error("The -c POS option not given\n");
+ if ( args->single_overlaps && args->merge_method_str ) error("The options --merge-logic and --single-overlaps cannot be combined\n");
+ if ( args->end_idx==-1 || (args->single_overlaps && !args->merge_method_str) )
+ {
+ args->end_idx = -args->beg_idx - 1;
+ args->tgts = bcf_sr_regions_init(args->targets_fname,1,args->chr_idx,args->beg_idx,args->end_idx);
+ if ( !args->tgts ) error("Could not initialize the annotation file: %s\n", args->targets_fname);
+ if ( !args->tgts->tbx ) error("Expected tabix-indexed annotation file: %s\n", args->targets_fname);
+ }
+ else
+ {
+ if ( args->ref_idx!=-1 ) error("Error: the REF columns will be ignored when BEG,END (or FROM,TO) is present. Replace END (or TO) with \"-\".\n");
+ int len = strlen(args->targets_fname);
+ if ( len>=7 && !strcasecmp(".bed.gz",args->targets_fname+len-7) ) args->tgt_is_bed = 1;
+ else if ( len>=8 && !strcasecmp(".bed.bgz",args->targets_fname+len-8) ) args->tgt_is_bed = 1;
+ else if ( len>=4 && !strcasecmp(".bed",args->targets_fname+len-4) ) args->tgt_is_bed = 1;
+ args->tgt_idx = regidx_init(args->targets_fname,parse_with_payload,free_payload,sizeof(char*),args);
+ if ( !args->tgt_idx ) error("Failed to parse: %s\n", args->targets_fname);
+ args->tgt_itr = regitr_init(args->tgt_idx);
+ args->nalines++;
+ hts_expand0(annot_line_t,args->nalines,args->malines,args->alines);
+ }
}
+ init_merge_method(args);
args->vcmp = vcmp_init();
if ( args->filter_str )
if ( args->rename_chrs ) rename_chrs(args, args->rename_chrs);
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
+ if ( args->out_fh == NULL ) error("[%s] Error: cannot write to \"%s\": %s\n", __func__,args->output_fname, strerror(errno));
if ( args->n_threads )
hts_set_opt(args->out_fh, HTS_OPT_THREAD_POOL, args->files->p);
- bcf_hdr_write(args->out_fh, args->hdr_out);
+ if ( bcf_hdr_write(args->out_fh, args->hdr_out)!=0 ) error("[%s] Error: failed to write the header to %s\n", __func__,args->output_fname);
}
}
{
free(args->cols[i].hdr_key_src);
free(args->cols[i].hdr_key_dst);
+ free(args->cols[i].mm_kstr.s);
+ if ( args->cols[i].mm_str_hash ) khash_str2int_destroy_free(args->cols[i].mm_str_hash);
+ free(args->cols[i].mm_dbl);
}
free(args->cols);
for (i=0; i<args->malines; i++)
free(args->alines[i].line.s);
}
free(args->alines);
+ if ( args->tgt_idx )
+ {
+ regidx_destroy(args->tgt_idx);
+ regitr_destroy(args->tgt_itr);
+ }
if ( args->tgts ) bcf_sr_regions_destroy(args->tgts);
free(args->tmpks.s);
free(args->tmpi);
free(args->sample_map);
}
+static void parse_annot_line(args_t *args, char *str, annot_line_t *tmp)
+{
+ tmp->line.l = 0;
+ kputs(str, &tmp->line);
+ char *s = tmp->line.s;
+ tmp->ncols = 1;
+ hts_expand(char*,tmp->ncols,tmp->mcols,tmp->cols);
+ tmp->cols[0] = s;
+ while ( *s )
+ {
+ if ( *s=='\t' )
+ {
+ tmp->ncols++;
+ hts_expand(char*,tmp->ncols,tmp->mcols,tmp->cols);
+ tmp->cols[tmp->ncols-1] = s+1;
+ *s = 0;
+ }
+ s++;
+ }
+ if ( args->ref_idx != -1 )
+ {
+ if ( args->ref_idx >= tmp->ncols )
+ error("Could not parse the line, expected %d+ columns, found %d:\n\t%s\n",args->ref_idx+1,tmp->ncols,str);
+ if ( args->alt_idx >= tmp->ncols )
+ error("Could not parse the line, expected %d+ columns, found %d:\n\t%s\n",args->alt_idx+1,tmp->ncols,str);
+ tmp->nals = 2;
+ hts_expand(char*,tmp->nals,tmp->mals,tmp->als);
+ tmp->als[0] = tmp->cols[args->ref_idx];
+ tmp->als[1] = s = tmp->cols[args->alt_idx];
+ while ( *s )
+ {
+ if ( *s==',' )
+ {
+ tmp->nals++;
+ hts_expand(char*,tmp->nals,tmp->mals,tmp->als);
+ tmp->als[tmp->nals-1] = s+1;
+ *s = 0;
+ }
+ s++;
+ }
+ }
+}
static void buffer_annot_lines(args_t *args, bcf1_t *line, int start_pos, int end_pos)
{
if ( args->nalines && args->alines[0].rid != line->rid ) args->nalines = 0;
tmp->rid = line->rid;
tmp->start = args->tgts->start;
tmp->end = args->tgts->end;
- tmp->line.l = 0;
- kputs(args->tgts->line.s, &tmp->line);
- char *s = tmp->line.s;
- tmp->ncols = 1;
- hts_expand(char*,tmp->ncols,tmp->mcols,tmp->cols);
- tmp->cols[0] = s;
- while ( *s )
- {
- if ( *s=='\t' )
- {
- tmp->ncols++;
- hts_expand(char*,tmp->ncols,tmp->mcols,tmp->cols);
- tmp->cols[tmp->ncols-1] = s+1;
- *s = 0;
- }
- s++;
- }
+ parse_annot_line(args, args->tgts->line.s, tmp);
if ( args->ref_idx != -1 )
{
- if ( args->ref_idx >= tmp->ncols )
- error("Could not parse the line, expected %d+ columns, found %d:\n\t%s\n",args->ref_idx+1,tmp->ncols,args->tgts->line.s);
- if ( args->alt_idx >= tmp->ncols )
- error("Could not parse the line, expected %d+ columns, found %d:\n\t%s\n",args->alt_idx+1,tmp->ncols,args->tgts->line.s);
- tmp->nals = 2;
- hts_expand(char*,tmp->nals,tmp->mals,tmp->als);
- tmp->als[0] = tmp->cols[args->ref_idx];
- tmp->als[1] = s = tmp->cols[args->alt_idx];
- while ( *s )
- {
- if ( *s==',' )
- {
- tmp->nals++;
- hts_expand(char*,tmp->nals,tmp->mals,tmp->als);
- tmp->als[tmp->nals-1] = s+1;
- *s = 0;
- }
- s++;
- }
int iseq = args->tgts->iseq;
if ( bcf_sr_regions_next(args->tgts)<0 || args->tgts->iseq!=iseq ) break;
}
for (i=0; i<args->nrm; i++)
args->rm[i].handler(args, line, &args->rm[i]);
- if ( args->tgts )
+ int has_overlap = 0;
+
+ if ( args->tgt_idx )
+ {
+ if ( regidx_overlap(args->tgt_idx, bcf_seqname(args->hdr,line),line->pos,line->pos+line->rlen-1, args->tgt_itr) )
+ {
+ while ( regitr_overlap(args->tgt_itr) )
+ {
+ annot_line_t *tmp = &args->alines[0];
+ tmp->rid = line->rid;
+ tmp->start = args->tgt_itr->beg;
+ tmp->end = args->tgt_itr->end;
+ parse_annot_line(args, regitr_payload(args->tgt_itr,char*), tmp);
+ for (j=0; j<args->ncols; j++)
+ if ( args->cols[j].setter(args,line,&args->cols[j],tmp) )
+ error("fixme: Could not set %s at %s:%"PRId64"\n", args->cols[j].hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
+ }
+ has_overlap = 1;
+ }
+ for (j=0; j<args->ncols; j++)
+ if ( args->cols[j].merge_method != MM_FIRST )
+ args->cols[j].setter(args,line,&args->cols[j],NULL);
+ }
+ else if ( args->tgts )
{
// Buffer annotation lines. When multiple ALT alleles are present in the
// annotation file, at least one must match one of the VCF alleles.
// there is a matching line
for (j=0; j<args->ncols; j++)
if ( args->cols[j].setter(args,line,&args->cols[j],&args->alines[i]) )
- error("fixme: Could not set %s at %s:%d\n", args->cols[j].hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
-
- }
-
- if ( args->mark_sites )
- {
- // ideally, we'd like to be far more general than this in future, see https://github.com/samtools/bcftools/issues/87
- if ( args->mark_sites_logic==MARK_LISTED )
- bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL,i<args->nalines?1:0);
- else
- bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL,i<args->nalines?0:1);
+ error("fixme: Could not set %s at %s:%"PRId64"\n", args->cols[j].hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
}
+ has_overlap = i<args->nalines ? 1 : 0;
}
else if ( args->files->nreaders == 2 )
{
bcf1_t *aline = bcf_sr_get_line(args->files,1);
for (j=0; j<args->ncols; j++)
if ( args->cols[j].setter(args,line,&args->cols[j],aline) )
- error("fixme: Could not set %s at %s:%d\n", args->cols[j].hdr_key_src,bcf_seqname(args->hdr,line),line->pos+1);
+ error("fixme: Could not set %s at %s:%"PRId64"\n", args->cols[j].hdr_key_src,bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
- if ( args->mark_sites )
- bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL,args->mark_sites_logic==MARK_LISTED ? 1 : 0);
+ has_overlap = 1;
}
- else if ( args->mark_sites )
- bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL, args->mark_sites_logic==MARK_UNLISTED ? 1 : 0);
}
if ( args->set_ids )
{
bcf_update_id(args->hdr_out,line,args->tmpks.s);
}
}
+
+ if ( args->mark_sites )
+ {
+ // ideally, we'd like to be far more general than this in future, see https://github.com/samtools/bcftools/issues/87
+ if ( args->mark_sites_logic==MARK_LISTED )
+ bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL,has_overlap?1:0);
+ else
+ bcf_update_info_flag(args->hdr_out,line,args->mark_sites,NULL,has_overlap?0:1);
+ }
}
static void usage(args_t *args)
fprintf(bcftools_stderr, " --collapse <string> matching records by <snps|indels|both|all|some|none>, see man page for details [some]\n");
fprintf(bcftools_stderr, " -c, --columns <list> list of columns in the annotation file, e.g. CHROM,POS,REF,ALT,-,INFO/TAG. See man page for details\n");
fprintf(bcftools_stderr, " -e, --exclude <expr> exclude sites for which the expression is true (see man page for details)\n");
+ fprintf(bcftools_stderr, " --force continue despite parsing error (at your own risk!)\n");
fprintf(bcftools_stderr, " -h, --header-lines <file> lines which should be appended to the VCF header\n");
fprintf(bcftools_stderr, " -I, --set-id [+]<format> set ID column, see man page for details\n");
fprintf(bcftools_stderr, " -i, --include <expr> select sites for which the expression is true (see man page for details)\n");
fprintf(bcftools_stderr, " -k, --keep-sites leave -i/-e sites unchanged instead of discarding them\n");
+ fprintf(bcftools_stderr, " -l, --merge-logic <tag:type> merge logic for multiple overlapping regions (see man page for details), EXPERIMENTAL\n");
fprintf(bcftools_stderr, " -m, --mark-sites [+-]<tag> add INFO/tag flag to sites which are (\"+\") or are not (\"-\") listed in the -a file\n");
fprintf(bcftools_stderr, " --no-version do not append version and command line to the header\n");
fprintf(bcftools_stderr, " -o, --output <file> write output to a file [standard output]\n");
fprintf(bcftools_stderr, " --rename-chrs <file> rename sequences according to map file: from\\tto\n");
fprintf(bcftools_stderr, " -s, --samples [^]<list> comma separated list of samples to annotate (or exclude with \"^\" prefix)\n");
fprintf(bcftools_stderr, " -S, --samples-file [^]<file> file of samples to annotate (or exclude with \"^\" prefix)\n");
+ fprintf(bcftools_stderr, " --single-overlaps keep memory low by avoiding complexities arising from handling multiple overlapping intervals\n");
fprintf(bcftools_stderr, " -x, --remove <list> list of annotations (e.g. ID,INFO/DP,FORMAT/DP,FILTER) to remove (or keep with \"^\" prefix). See man page for details\n");
fprintf(bcftools_stderr, " --threads <int> number of extra output compression threads [0]\n");
fprintf(bcftools_stderr, "\n");
args->output_type = FT_VCF;
args->n_threads = 0;
args->record_cmd_line = 1;
- args->ref_idx = args->alt_idx = args->chr_idx = args->from_idx = args->to_idx = -1;
+ args->ref_idx = args->alt_idx = args->chr_idx = args->beg_idx = args->end_idx = -1;
args->set_ids_replace = 1;
int regions_is_file = 0, collapse = 0;
static struct option loptions[] =
{
- {"keep-sites",required_argument,NULL,'k'},
+ {"keep-sites",no_argument,NULL,'k'},
{"mark-sites",required_argument,NULL,'m'},
{"set-id",required_argument,NULL,'I'},
{"output",required_argument,NULL,'o'},
{"output-type",required_argument,NULL,'O'},
{"threads",required_argument,NULL,9},
{"annotations",required_argument,NULL,'a'},
+ {"merge-logic",required_argument,NULL,'l'},
{"collapse",required_argument,NULL,2},
{"include",required_argument,NULL,'i'},
{"exclude",required_argument,NULL,'e'},
{"header-lines",required_argument,NULL,'h'},
{"samples",required_argument,NULL,'s'},
{"samples-file",required_argument,NULL,'S'},
+ {"single-overlaps",no_argument,NULL,10},
{"no-version",no_argument,NULL,8},
+ {"force",no_argument,NULL,'f'},
{NULL,0,NULL,0}
};
- while ((c = getopt_long(argc, argv, "h:?o:O:r:R:a:x:c:i:e:S:s:I:m:k",loptions,NULL)) >= 0)
+ while ((c = getopt_long(argc, argv, "h:?o:O:r:R:a:x:c:i:e:S:s:I:m:kl:f",loptions,NULL)) >= 0)
{
switch (c) {
+ case 'f': args->force = 1; break;
case 'k': args->keep_sites = 1; break;
case 'm':
args->mark_sites_logic = MARK_LISTED;
else if ( optarg[0]=='-' ) { args->mark_sites = optarg+1; args->mark_sites_logic = MARK_UNLISTED; }
else args->mark_sites = optarg;
break;
+ case 'l': args->merge_method_str = optarg; break;
case 'I': args->set_ids_fmt = optarg; break;
case 's': args->sample_names = optarg; break;
case 'S': args->sample_names = optarg; args->sample_is_file = 1; break;
break;
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
+ case 10 : args->single_overlaps = 1; break;
case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
if ( args->targets_fname )
{
htsFile *fp = hts_open(args->targets_fname,"r");
+ if ( !fp ) error("Failed to open %s\n", args->targets_fname);
htsFormat type = *hts_get_format(fp);
hts_close(fp);
}
}
if ( bcf_sr_set_threads(args->files, args->n_threads)<0 ) error("Failed to create threads\n");
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
+ static int line_errcode_warned = 0;
init_data(args);
while ( bcf_sr_next_line(args->files) )
{
if ( !bcf_sr_has_line(args->files,0) ) continue;
bcf1_t *line = bcf_sr_get_line(args->files,0);
- if ( line->errcode ) error("Encountered error, cannot proceed. Please check the error output above.\n");
+ if ( line->errcode )
+ {
+ if ( !args->force )
+ error("Encountered an error, cannot proceed. Please check the error output above.\n"
+ "If feeling adventurous, use the --force option. (At your own risk!)\n");
+ else if ( !line_errcode_warned )
+ {
+ fprintf(bcftools_stderr,
+ "Warning: Encountered an error, proceeding only because --force was given.\n"
+ " Note that this can result in a segfault or a silent corruption of the output file!\n");
+ line_errcode_warned = 1;
+ line->errcode = 0;
+ }
+ }
if ( args->filter )
{
int pass = filter_test(args->filter, line, NULL);
if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
if ( !pass )
{
- if ( args->keep_sites ) bcf_write1(args->out_fh, args->hdr_out, line);
+ if ( args->keep_sites && bcf_write1(args->out_fh, args->hdr_out, line)!=0 ) error("[%s] Error: failed to write to %s\n", __func__,args->output_fname);
continue;
}
}
annotate(args, line);
- bcf_write1(args->out_fh, args->hdr_out, line);
+ if ( bcf_write1(args->out_fh, args->hdr_out, line)!=0 ) error("[%s] Error: failed to write to %s\n", __func__,args->output_fname);
}
destroy_data(args);
bcf_sr_destroy(args->files);
/* The MIT License
- Copyright (c) 2016 Genome Research Ltd.
+ Copyright (c) 2016-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
}
prune_t;
+typedef struct
+{
+ int active;
+}
+rmdup_t;
+
typedef struct
{
int active, rid, end;
ld_t ld;
prune_t prune;
overlap_t overlap;
+ rmdup_t rmdup;
};
vcfbuf_t *vcfbuf_init(bcf_hdr_t *hdr, int win)
if ( key==VCFBUF_NSITES ) { buf->prune.max_sites = *((int*)value); return; }
if ( key==VCFBUF_AF_TAG ) { buf->prune.af_tag = *((char**)value); return; }
if ( key==VCFBUF_OVERLAP_WIN ) { buf->overlap.active = *((int*)value); return; }
+ if ( key==VCFBUF_RMDUP) { buf->rmdup.active = *((int*)value); return; }
}
int vcfbuf_nsites(vcfbuf_t *buf)
return ret;
}
+bcf1_t *vcfbuf_peek(vcfbuf_t *buf, int idx)
+{
+ int i = rbuf_kth(&buf->rbuf, idx);
+ return i<0 ? NULL : buf->vcf[i].rec;
+}
+
+bcf1_t *vcfbuf_remove(vcfbuf_t *buf, int idx)
+{
+ int i = rbuf_kth(&buf->rbuf, idx);
+ if ( i<0 ) return NULL;
+ bcf1_t *rec = buf->vcf[i].rec;
+ rbuf_remove_kth(&buf->rbuf, vcfrec_t, idx, buf->vcf);
+ return rec;
+}
+
static int cmpvrec(const void *_a, const void *_b)
{
vcfrec_t *a = *((vcfrec_t**) _a);
rbuf_remove_kth(&buf->rbuf, vcfrec_t, buf->prune.idx[i], buf->vcf);
}
+static int _rmdup_can_flush(vcfbuf_t *buf, int flush_all)
+{
+ if ( flush_all ) return 1;
+
+ if ( buf->rbuf.n==1 ) return 0;
+
+ int k1 = rbuf_kth(&buf->rbuf, -1);
+ int k2 = rbuf_kth(&buf->rbuf, -2);
+
+ vcfrec_t *rec1 = &buf->vcf[k1];
+ vcfrec_t *rec2 = &buf->vcf[k2];
+
+ if ( rec1->rec->rid!=rec2->rec->rid ) return 1;
+ if ( rec1->rec->pos!=rec2->rec->pos ) return 1;
+
+ return 0;
+}
+
static int _overlap_can_flush(vcfbuf_t *buf, int flush_all)
{
if ( flush_all ) { buf->overlap.rid = -1; return 1; }
j = rbuf_last(&buf->rbuf); // last
if ( buf->vcf[i].rec->rid != buf->vcf[j].rec->rid ) goto ret;
- if ( buf->overlap.active )
- {
- int ret = _overlap_can_flush(buf, flush_all);
- //printf("can_flush: %d %d - %d\n", ret, buf->vcf[i].rec->pos+1, buf->vcf[j].rec->pos+1);
- if ( ret ) goto ret;
- }
- //if ( buf->overlap.active && _overlap_can_flush(buf, flush_all) ) goto ret;
+ if ( buf->overlap.active && _overlap_can_flush(buf, flush_all) ) goto ret;
+ if ( buf->rmdup.active && _rmdup_can_flush(buf, flush_all) ) goto ret;
if ( buf->win > 0 )
{
/* The MIT License
- Copyright (c) 2016 Genome Research Ltd.
+ Copyright (c) 2016-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
}
prune_t;
+typedef struct
+{
+ int active;
+}
+rmdup_t;
+
typedef struct
{
int active, rid, end;
ld_t ld;
prune_t prune;
overlap_t overlap;
+ rmdup_t rmdup;
};
vcfbuf_t *vcfbuf_init(bcf_hdr_t *hdr, int win)
if ( key==VCFBUF_NSITES ) { buf->prune.max_sites = *((int*)value); return; }
if ( key==VCFBUF_AF_TAG ) { buf->prune.af_tag = *((char**)value); return; }
if ( key==VCFBUF_OVERLAP_WIN ) { buf->overlap.active = *((int*)value); return; }
+ if ( key==VCFBUF_RMDUP) { buf->rmdup.active = *((int*)value); return; }
}
int vcfbuf_nsites(vcfbuf_t *buf)
return ret;
}
+bcf1_t *vcfbuf_peek(vcfbuf_t *buf, int idx)
+{
+ int i = rbuf_kth(&buf->rbuf, idx);
+ return i<0 ? NULL : buf->vcf[i].rec;
+}
+
+bcf1_t *vcfbuf_remove(vcfbuf_t *buf, int idx)
+{
+ int i = rbuf_kth(&buf->rbuf, idx);
+ if ( i<0 ) return NULL;
+ bcf1_t *rec = buf->vcf[i].rec;
+ rbuf_remove_kth(&buf->rbuf, vcfrec_t, idx, buf->vcf);
+ return rec;
+}
+
static int cmpvrec(const void *_a, const void *_b)
{
vcfrec_t *a = *((vcfrec_t**) _a);
rbuf_remove_kth(&buf->rbuf, vcfrec_t, buf->prune.idx[i], buf->vcf);
}
+static int _rmdup_can_flush(vcfbuf_t *buf, int flush_all)
+{
+ if ( flush_all ) return 1;
+
+ if ( buf->rbuf.n==1 ) return 0;
+
+ int k1 = rbuf_kth(&buf->rbuf, -1);
+ int k2 = rbuf_kth(&buf->rbuf, -2);
+
+ vcfrec_t *rec1 = &buf->vcf[k1];
+ vcfrec_t *rec2 = &buf->vcf[k2];
+
+ if ( rec1->rec->rid!=rec2->rec->rid ) return 1;
+ if ( rec1->rec->pos!=rec2->rec->pos ) return 1;
+
+ return 0;
+}
+
static int _overlap_can_flush(vcfbuf_t *buf, int flush_all)
{
if ( flush_all ) { buf->overlap.rid = -1; return 1; }
j = rbuf_last(&buf->rbuf); // last
if ( buf->vcf[i].rec->rid != buf->vcf[j].rec->rid ) goto ret;
- if ( buf->overlap.active )
- {
- int ret = _overlap_can_flush(buf, flush_all);
- //printf("can_flush: %d %d - %d\n", ret, buf->vcf[i].rec->pos+1, buf->vcf[j].rec->pos+1);
- if ( ret ) goto ret;
- }
- //if ( buf->overlap.active && _overlap_can_flush(buf, flush_all) ) goto ret;
+ if ( buf->overlap.active && _overlap_can_flush(buf, flush_all) ) goto ret;
+ if ( buf->rmdup.active && _rmdup_can_flush(buf, flush_all) ) goto ret;
if ( buf->win > 0 )
{
/* The MIT License
- Copyright (c) 2017 Genome Research Ltd.
+ Copyright (c) 2017-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
VCFBUF_NSITES, // leave at max this many sites in the window
VCFBUF_AF_TAG, // use this INFO tag with LD_NSITES
VCFBUF_OVERLAP_WIN, // keep only overlapping variants in the window
+ VCFBUF_RMDUP, // remove duplicate sites (completely)
}
vcfbuf_opt_t;
*/
bcf1_t *vcfbuf_push(vcfbuf_t *buf, bcf1_t *rec, int swap);
+/*
+ * vcfbuf_peek() - return pointer to i-th record in the buffer but do not remove it from the buffer
+ * @idx: 0-based index to buffered lines
+ */
+bcf1_t *vcfbuf_peek(vcfbuf_t *buf, int idx);
+
+/*
+ * vcfbuf_remove() - return pointer to i-th record in the buffer and remove it from the buffer
+ * @idx: 0-based index to buffered lines
+ */
+bcf1_t *vcfbuf_remove(vcfbuf_t *buf, int idx);
+
bcf1_t *vcfbuf_flush(vcfbuf_t *buf, int flush_all);
/*
#include "prob1.h"
#include "ploidy.h"
#include "gvcf.h"
+#include "regidx.h"
+#include "vcfbuf.h"
void error(const char *format, ...);
-#ifdef _WIN32
-#define srand48(x) srand(x)
-#define lrand48() rand()
-#endif
-
#define CF_NO_GENO 1
#define CF_INS_MISSED (1<<1)
#define CF_CCALL (1<<2)
#define CF_QCNT (1<<13)
#define CF_INDEL_ONLY (1<<14)
+typedef struct
+{
+ tgt_als_t *als;
+ int nmatch_als, ibuf;
+}
+rec_tgt_t;
+
typedef struct
{
int flag; // combination of CF_* flags above
int nsamples, *samples_map; // mapping from output sample names to original VCF
char *regions, *targets; // regions to process
int regions_is_file, targets_is_file;
+ regidx_t *tgt_idx;
+ regitr_t *tgt_itr, *tgt_itr_prev, *tgt_itr_tmp;
+ vcfbuf_t *vcfbuf;
char *samples_fname;
int samples_is_file;
bcf1_t *missed_line;
call_t aux; // parameters and temporary data
+ kstring_t str;
int argc;
char **argv;
if ( ismpl < 0 ) { fprintf(stderr,"Warning: No such sample in the VCF: %s\n",ss); continue; }
if ( old2new[ismpl] != -1 ) { fprintf(stderr,"Warning: The sample is listed multiple times: %s\n",ss); continue; }
- ss = se+1;
+ ss = se+(x != '\0');
while ( *ss && isspace(*ss) ) ss++;
if ( !*ss ) ss = "2"; // default ploidy
se = ss;
bcf_float_set_missing(args->missed_line->qual);
}
-static void print_missed_line(bcf_sr_regions_t *regs, void *data)
+static int tgt_parse(const char *line, char **chr_beg, char **chr_end, uint32_t *beg, uint32_t *end, void *payload, void *usr)
{
- args_t *args = (args_t*) data;
- call_t *call = &args->aux;
- bcf1_t *missed = args->missed_line;
+ char *ss = (char*) line;
+ while ( *ss && isspace(*ss) ) ss++;
+ if ( !*ss ) { fprintf(stderr,"Could not parse the line: %s\n", line); return -2; }
+ if ( *ss=='#' ) return -1; // skip comments
- char *ss = regs->line.s;
- int i = 0;
- while ( i<args->aux.srs->targets_als-1 && *ss )
+ char *se = ss;
+ while ( *se && !isspace(*se) ) se++;
+
+ *chr_beg = ss;
+ *chr_end = se-1;
+
+ if ( !*se ) { fprintf(stderr,"Could not parse the line: %s\n", line); return -2; }
+
+ ss = se+1;
+ *beg = strtod(ss, &se);
+ if ( ss==se ) { fprintf(stderr,"Could not parse tab line: %s\n", line); return -2; }
+ if ( *beg==0 ) { fprintf(stderr,"Could not parse tab line, expected 1-based coordinate: %s\n", line); return -2; }
+ (*beg)--;
+ *end = *beg;
+
+ if ( !usr ) return 0; // allele information not required
+
+ ss = se+1;
+ tgt_als_t *als = (tgt_als_t*)payload;
+ als->used = 0;
+ als->n = 0;
+ als->allele = NULL;
+ while ( *ss )
{
- if ( *ss=='\t' ) i++;
- ss++;
+ se = ss;
+ while ( *se && *se!=',' ) se++;
+ als->n++;
+ als->allele = (char**)realloc(als->allele,als->n*sizeof(*als->allele));
+ als->allele[als->n-1] = (char*)malloc(se-ss+1);
+ memcpy(als->allele[als->n-1],ss,se-ss);
+ als->allele[als->n-1][se-ss] = 0;
+ ss = se+1;
+ if ( !*se ) break;
}
- if ( !*ss ) error("Could not parse: [%s] (%d)\n", regs->line.s,args->aux.srs->targets_als);
+ return 0;
+}
+static void tgt_free(void *payload)
+{
+ tgt_als_t *als = (tgt_als_t*)payload;
+ int i;
+ for (i=0; i<als->n; i++) free(als->allele[i]);
+ free(als->allele);
+}
+static void tgt_flush_region(args_t *args, char *chr, uint32_t beg, uint32_t end)
+{
+ if ( !regidx_overlap(args->tgt_idx, chr,beg,end,args->tgt_itr_tmp) ) return;
+ while ( regitr_overlap(args->tgt_itr_tmp) )
+ {
+ if ( args->tgt_itr_tmp->beg < beg ) continue;
- missed->rid = bcf_hdr_name2id(call->hdr,regs->seq_names[regs->prev_seq]);
- missed->pos = regs->start;
- bcf_update_alleles_str(call->hdr, missed,ss);
+ tgt_als_t *tgt_als = ®itr_payload(args->tgt_itr_tmp,tgt_als_t);
+ if ( tgt_als->used ) continue;
- bcf_write1(args->out_fh, call->hdr, missed);
+ args->missed_line->rid = bcf_hdr_name2id(args->aux.hdr,chr);
+ args->missed_line->pos = args->tgt_itr_tmp->beg;
+ bcf_unpack(args->missed_line,BCF_UN_ALL);
+ bcf_update_alleles(args->aux.hdr, args->missed_line, (const char**)tgt_als->allele, tgt_als->n);
+ tgt_als->used = 1;
+ if ( bcf_write1(args->out_fh, args->aux.hdr, args->missed_line)!=0 ) error("[%s] Error: failed to write to %s\n", __func__,args->output_fname);
+ }
+}
+static void tgt_flush(args_t *args, bcf1_t *rec)
+{
+ if ( rec )
+ {
+ char *chr = (char*)bcf_seqname(args->aux.hdr,rec);
+
+ if ( !args->tgt_itr_prev ) // first record
+ tgt_flush_region(args,chr,0,rec->pos-1);
+
+ else if ( strcmp(chr,args->tgt_itr_prev->seq) ) // first record on a new chromosome
+ {
+ tgt_flush_region(args,args->tgt_itr_prev->seq,args->tgt_itr_prev->beg+1,REGIDX_MAX);
+ tgt_flush_region(args,chr,0,rec->pos-1);
+ }
+ else // another record on the same chromosome
+ tgt_flush_region(args,args->tgt_itr_prev->seq,args->tgt_itr_prev->beg,rec->pos-1);
+ }
+ else
+ {
+ // flush everything
+ if ( args->tgt_itr_prev )
+ tgt_flush_region(args,args->tgt_itr_prev->seq,args->tgt_itr_prev->beg,REGIDX_MAX);
+
+ int i, nchr = 0;
+ char **chr = regidx_seq_names(args->tgt_idx, &nchr);
+ for (i=0; i<nchr; i++)
+ tgt_flush_region(args,chr[i],0,REGIDX_MAX);
+ }
+}
+inline static int is_indel(int nals, char **als)
+{
+ // This is mpileup output, we can make some assumption:
+ // - no MNPs
+ // - "<*>" is not present at indels sites and there are no other symbolic alleles than <*>
+ if ( als[1][0]=='<' ) return 0;
+
+ int i;
+ for (i=0; i<nals; i++)
+ {
+ if ( als[i][0]=='<' ) continue;
+ if ( als[i][1] ) return 1;
+ }
+ return 0;
+}
+bcf1_t *next_line(args_t *args)
+{
+ bcf1_t *rec = NULL;
+ if ( !args->vcfbuf )
+ {
+ while ( bcf_sr_next_line(args->aux.srs) )
+ {
+ rec = args->aux.srs->readers[0].buffer[0];
+ if ( args->aux.srs->errnum || rec->errcode ) error("Error: could not parse the input VCF\n");
+ if ( args->tgt_idx )
+ {
+ if ( !regidx_overlap(args->tgt_idx, bcf_seqname(args->aux.hdr,rec),rec->pos,rec->pos,args->tgt_itr) ) continue;
+
+ // For backward compatibility: require the exact position, not an interval overlap
+ int pos_match = 0;
+ while ( regitr_overlap(args->tgt_itr) )
+ {
+ if ( args->tgt_itr->beg != rec->pos ) continue;
+ pos_match = 1;
+ break;
+ }
+ if ( !pos_match ) continue;
+ }
+ if ( args->samples_map ) bcf_subset(args->aux.hdr, rec, args->nsamples, args->samples_map);
+ bcf_unpack(rec, BCF_UN_STR);
+ return rec;
+ }
+ return NULL;
+ }
+
+ // If we are here,-C alleles was given and vcfbuf and tgt_idx are set
+
+ // Fill the buffer with duplicate lines
+ int vcfbuf_full = 1;
+ int nbuf = vcfbuf_nsites(args->vcfbuf);
+ bcf1_t *rec0 = NULL, *recN = NULL;
+ if ( nbuf==0 ) vcfbuf_full = 0;
+ else if ( nbuf==1 )
+ {
+ vcfbuf_full = 0;
+ rec0 = vcfbuf_peek(args->vcfbuf, 0);
+ }
+ else
+ {
+ rec0 = vcfbuf_peek(args->vcfbuf, 0);
+ recN = vcfbuf_peek(args->vcfbuf, nbuf-1);
+ if ( rec0->rid == recN->rid && rec0->pos == recN->pos ) vcfbuf_full = 0;
+ }
+ if ( !vcfbuf_full )
+ {
+ while ( bcf_sr_next_line(args->aux.srs) )
+ {
+ rec = args->aux.srs->readers[0].buffer[0];
+ if ( args->aux.srs->errnum || rec->errcode ) error("Error: could not parse the input VCF\n");
+ if ( !regidx_overlap(args->tgt_idx, bcf_seqname(args->aux.hdr,rec),rec->pos,rec->pos,args->tgt_itr) ) continue;
+ // as above: require the exact position, not an interval overlap
+ int exact_match = 0;
+ while ( regitr_overlap(args->tgt_itr) )
+ {
+ if ( args->tgt_itr->beg != rec->pos ) continue;
+ exact_match = 1;
+ break;
+ }
+ if ( !exact_match ) continue;
+
+ if ( args->samples_map ) bcf_subset(args->aux.hdr, rec, args->nsamples, args->samples_map);
+ bcf_unpack(rec, BCF_UN_STR);
+ if ( !rec0 ) rec0 = rec;
+ recN = rec;
+ args->aux.srs->readers[0].buffer[0] = vcfbuf_push(args->vcfbuf, rec, 1);
+ if ( rec0->rid!=recN->rid || rec0->pos!=recN->pos ) break;
+ }
+ }
+
+ nbuf = vcfbuf_nsites(args->vcfbuf);
+ int n, i,j;
+ for (n=nbuf; n>1; n--)
+ {
+ recN = vcfbuf_peek(args->vcfbuf, n-1);
+ if ( rec0->rid==recN->rid && rec0->pos==recN->pos ) break;
+ }
+ if ( n==0 )
+ {
+ assert( !nbuf );
+ return NULL;
+ }
+
+ // Find the VCF and tab record with the best matching combination of alleles, prioritize
+ // records of the same type (snp vs indel)
+ rec_tgt_t rec_tgt;
+ memset(&rec_tgt,0,sizeof(rec_tgt));
+ regidx_overlap(args->tgt_idx, bcf_seqname(args->aux.hdr,rec0),rec0->pos,rec0->pos,args->tgt_itr);
+ regitr_t *tmp_itr = regitr_init(args->tgt_idx);
+ regitr_copy(tmp_itr, args->tgt_itr);
+ for (i=0; i<n; i++)
+ {
+ rec = vcfbuf_peek(args->vcfbuf, i);
+ int rec_indel = is_indel(rec->n_allele, rec->d.allele) ? 1 : -1;
+ while ( regitr_overlap(tmp_itr) )
+ {
+ if ( tmp_itr->beg != rec->pos ) continue;
+ tgt_als_t *als = ®itr_payload(tmp_itr,tgt_als_t);
+ if ( als->used ) continue;
+ int nmatch_als = 0;
+ vcmp_t *vcmp = vcmp_init();
+ int ret = vcmp_set_ref(vcmp, rec->d.allele[0], als->allele[0]);
+ if ( ret==0 )
+ {
+ nmatch_als++;
+ if ( rec->n_allele > 1 && als->n > 1 )
+ {
+ for (j=1; j<als->n; j++)
+ {
+ if ( vcmp_find_allele(vcmp, rec->d.allele+1, rec->n_allele-1, als->allele[j])>=0 ) nmatch_als++;
+ }
+ }
+ }
+ int als_indel = is_indel(als->n, als->allele) ? 1 : -1;
+ nmatch_als *= rec_indel*als_indel;
+ if ( nmatch_als > rec_tgt.nmatch_als || !rec_tgt.als )
+ {
+ rec_tgt.nmatch_als = nmatch_als;
+ rec_tgt.als = als;
+ rec_tgt.ibuf = i;
+ }
+ vcmp_destroy(vcmp);
+ }
+ }
+ regitr_destroy(tmp_itr);
+
+ args->aux.tgt_als = rec_tgt.als;
+ if ( rec_tgt.als ) rec_tgt.als->used = 1;
+
+ rec = vcfbuf_remove(args->vcfbuf, rec_tgt.ibuf);
+ return rec;
}
static void init_data(args_t *args)
// Open files for input and output, initialize structures
if ( args->targets )
{
- if ( bcf_sr_set_targets(args->aux.srs, args->targets, args->targets_is_file, args->aux.flag&CALL_CONSTR_ALLELES ? 3 : 0)<0 )
- error("Failed to read the targets: %s\n", args->targets);
-
- if ( args->aux.flag&CALL_CONSTR_ALLELES && args->flag&CF_INS_MISSED )
- {
- args->aux.srs->targets->missed_reg_handler = print_missed_line;
- args->aux.srs->targets->missed_reg_data = args;
- }
+ args->tgt_idx = regidx_init(args->targets, tgt_parse, args->aux.flag&CALL_CONSTR_ALLELES ? tgt_free : NULL, sizeof(tgt_als_t), args->aux.flag&CALL_CONSTR_ALLELES ? args : NULL);
+ args->tgt_itr = regitr_init(args->tgt_idx);
+ args->tgt_itr_tmp = regitr_init(args->tgt_idx);
}
+
if ( args->regions )
{
if ( bcf_sr_set_regions(args->aux.srs, args->regions, args->regions_is_file)<0 )
error("Failed to read the regions: %s\n", args->regions);
}
- if ( !bcf_sr_add_reader(args->aux.srs, args->bcf_fname) ) error("Failed to open %s: %s\n", args->bcf_fname,bcf_sr_strerror(args->aux.srs->errnum));
+ if ( !bcf_sr_add_reader(args->aux.srs, args->bcf_fname) )
+ error("Failed to read from %s: %s\n", !strcmp("-",args->bcf_fname)?"standard input":args->bcf_fname,bcf_sr_strerror(args->aux.srs->errnum));
args->aux.hdr = bcf_sr_get_header(args->aux.srs,0);
int i;
}
}
+ if ( args->aux.flag & CALL_CONSTR_ALLELES )
+ args->vcfbuf = vcfbuf_init(args->aux.hdr, 0);
+
args->out_fh = hts_open(args->output_fname, hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
+ if ( args->out_fh == NULL ) error("Error: cannot write to \"%s\": %s\n", args->output_fname, strerror(errno));
if ( args->n_threads ) hts_set_threads(args->out_fh, args->n_threads);
if ( args->flag & CF_QCALL )
bcf_hdr_remove(args->aux.hdr, BCF_HL_INFO, "I16");
if (args->record_cmd_line) bcf_hdr_append_version(args->aux.hdr, args->argc, args->argv, "bcftools_call");
- bcf_hdr_write(args->out_fh, args->aux.hdr);
+ if ( bcf_hdr_write(args->out_fh, args->aux.hdr)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname);
if ( args->flag&CF_INS_MISSED ) init_missed_line(args);
}
static void destroy_data(args_t *args)
{
+ if ( args->vcfbuf ) vcfbuf_destroy(args->vcfbuf);
+ if ( args->tgt_idx )
+ {
+ regidx_destroy(args->tgt_idx);
+ regitr_destroy(args->tgt_itr);
+ regitr_destroy(args->tgt_itr_tmp);
+ if ( args->tgt_itr_prev ) regitr_destroy(args->tgt_itr_prev);
+ }
if ( args->flag & CF_CCALL ) ccall_destroy(&args->aux);
else if ( args->flag & CF_MCALL ) mcall_destroy(&args->aux);
else if ( args->flag & CF_QCALL ) qcall_destroy(&args->aux);
free(args->samples_map);
free(args->sample2sex);
free(args->aux.ploidy);
+ free(args->str.s);
if ( args->gvcf ) gvcf_destroy(args->gvcf);
bcf_hdr_destroy(args->aux.hdr);
- hts_close(args->out_fh);
+ if ( hts_close(args->out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
bcf_sr_destroy(args->aux.srs);
}
static void usage(args_t *args)
{
fprintf(stderr, "\n");
- fprintf(stderr, "About: SNP/indel variant calling from VCF/BCF. To be used in conjunction with samtools mpileup.\n");
+ fprintf(stderr, "About: SNP/indel variant calling from VCF/BCF. To be used in conjunction with bcftools mpileup.\n");
fprintf(stderr, " This command replaces the former \"bcftools view\" caller. Some of the original\n");
fprintf(stderr, " functionality has been temporarily lost in the process of transition to htslib,\n");
fprintf(stderr, " but will be added back on popular demand. The original calling model can be\n");
fprintf(stderr, " -S, --samples-file <file> PED file or a file with an optional column with sex (see man page for details) [all samples]\n");
fprintf(stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
- fprintf(stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(stderr, "\n");
fprintf(stderr, "Input/output options:\n");
fprintf(stderr, " -A, --keep-alts keep all possible alternate alleles at variant sites\n");
fprintf(stderr, " -f, --format-fields <list> output format fields: GQ,GP (lowercase allowed) []\n");
fprintf(stderr, " -F, --prior-freqs <AN,AC> use prior allele frequencies\n");
+ fprintf(stderr, " -G, --group-samples <file|-> group samples by population (file with \"sample\\tgroup\") or \"-\" for single-sample calling\n");
fprintf(stderr, " -g, --gvcf <int>,[...] group non-variant sites into gVCF blocks by minimum per-sample DP\n");
fprintf(stderr, " -i, --insert-missed output also sites missed by mpileup but present in -T\n");
fprintf(stderr, " -M, --keep-masked-ref keep sites with masked reference allele (REF=N)\n");
fprintf(stderr, " -n, --novel-rate <float>,[...] likelihood of novel mutation for constrained trio calling, see man page for details [1e-8,1e-9,1e-9]\n");
fprintf(stderr, " -p, --pval-threshold <float> variant if P(ref|D)<FLOAT with -c [0.5]\n");
fprintf(stderr, " -P, --prior <float> mutation rate (use bigger for greater sensitivity), use with -m [1.1e-3]\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Example:\n");
+ fprintf(stderr, " # See also http://samtools.github.io/bcftools/howtos/variant-calling.html\n");
+ fprintf(stderr, " bcftools mpileup -f reference.fa alignments.bam | bcftools call -mv -Ob -o calls.bcf\n");
// todo (and more)
// fprintf(stderr, "\nContrast calling and association test options:\n");
{"format-fields",required_argument,NULL,'f'},
{"prior-freqs",required_argument,NULL,'F'},
{"gvcf",required_argument,NULL,'g'},
+ {"group-samples",required_argument,NULL,'G'},
{"output",required_argument,NULL,'o'},
{"output-type",required_argument,NULL,'O'},
{"regions",required_argument,NULL,'r'},
};
char *tmp = NULL;
- while ((c = getopt_long(argc, argv, "h?o:O:r:R:s:S:t:T:ANMV:vcmp:C:n:P:f:ig:XYF:", loptions, NULL)) >= 0)
+ while ((c = getopt_long(argc, argv, "h?o:O:r:R:s:S:t:T:ANMV:vcmp:C:n:P:f:ig:XYF:G:", loptions, NULL)) >= 0)
{
switch (c)
{
case 1 : ploidy = optarg; break;
case 'X': ploidy = "X"; fprintf(stderr,"Warning: -X will be deprecated, please use --ploidy instead.\n"); break;
case 'Y': ploidy = "Y"; fprintf(stderr,"Warning: -Y will be deprecated, please use --ploidy instead.\n"); break;
+ case 'G': args.aux.sample_groups = optarg; break;
case 'f': args.aux.output_tags |= parse_format_flag(optarg); break;
case 'M': args.flag &= ~CF_ACGT_ONLY; break; // keep sites where REF is N
case 'N': args.flag |= CF_ACGT_ONLY; break; // omit sites where first base in REF is N (the new default)
}
if ( args.flag & CF_INS_MISSED && !(args.aux.flag&CALL_CONSTR_ALLELES) ) error("The -i option requires -C alleles\n");
if ( args.aux.flag&CALL_VARONLY && args.gvcf ) error("The two options cannot be combined: --variants-only and --gvcf\n");
+ if ( args.aux.sample_groups && !(args.flag & CF_MCALL) ) error("The -G feature is supported only with the -m calling mode\n");
init_data(&args);
- while ( bcf_sr_next_line(args.aux.srs) )
+ bcf1_t *bcf_rec;
+ while ( (bcf_rec = next_line(&args)) )
{
- bcf1_t *bcf_rec = args.aux.srs->readers[0].buffer[0];
- if ( args.samples_map ) bcf_subset(args.aux.hdr, bcf_rec, args.nsamples, args.samples_map);
- bcf_unpack(bcf_rec, BCF_UN_STR);
+ // Skip duplicate positions with all matching `-C alleles -T` used up
+ if ( args.aux.flag&CALL_CONSTR_ALLELES && !args.aux.tgt_als ) continue;
// Skip unwanted sites
int i, is_indel = bcf_is_snp(bcf_rec) ? 0 : 1;
continue;
}
+ if ( args.flag & CF_INS_MISSED )
+ {
+ tgt_flush(&args,bcf_rec);
+ if ( !args.tgt_itr_prev ) args.tgt_itr_prev = regitr_init(args.tgt_idx);
+ regitr_copy(args.tgt_itr_prev, args.tgt_itr);
+ }
+
// Calling modes which output VCFs
int ret;
if ( args.flag & CF_MCALL )
if ( (args.aux.flag & CALL_VARONLY) && ret==0 && !args.gvcf ) continue; // not a variant
if ( args.gvcf )
bcf_rec = gvcf_write(args.gvcf, args.out_fh, args.aux.hdr, bcf_rec, ret==1?1:0);
- if ( bcf_rec )
- bcf_write1(args.out_fh, args.aux.hdr, bcf_rec);
+ if ( bcf_rec && bcf_write1(args.out_fh, args.aux.hdr, bcf_rec)!=0 ) error("[%s] Error: failed to write to %s\n", __func__,args.output_fname);
}
if ( args.gvcf ) gvcf_write(args.gvcf, args.out_fh, args.aux.hdr, NULL, 0);
- if ( args.flag & CF_INS_MISSED ) bcf_sr_regions_flush(args.aux.srs->targets);
+ if ( args.flag & CF_INS_MISSED ) tgt_flush(&args,NULL);
destroy_data(&args);
return 0;
}
#include "prob1.h"
#include "ploidy.h"
#include "gvcf.h"
+#include "regidx.h"
+#include "vcfbuf.h"
void error(const char *format, ...);
-#ifdef _WIN32
-#define srand48(x) srand(x)
-#define lrand48() rand()
-#endif
-
#define CF_NO_GENO 1
#define CF_INS_MISSED (1<<1)
#define CF_CCALL (1<<2)
#define CF_QCNT (1<<13)
#define CF_INDEL_ONLY (1<<14)
+typedef struct
+{
+ tgt_als_t *als;
+ int nmatch_als, ibuf;
+}
+rec_tgt_t;
+
typedef struct
{
int flag; // combination of CF_* flags above
int nsamples, *samples_map; // mapping from output sample names to original VCF
char *regions, *targets; // regions to process
int regions_is_file, targets_is_file;
+ regidx_t *tgt_idx;
+ regitr_t *tgt_itr, *tgt_itr_prev, *tgt_itr_tmp;
+ vcfbuf_t *vcfbuf;
char *samples_fname;
int samples_is_file;
bcf1_t *missed_line;
call_t aux; // parameters and temporary data
+ kstring_t str;
int argc;
char **argv;
if ( ismpl < 0 ) { fprintf(bcftools_stderr,"Warning: No such sample in the VCF: %s\n",ss); continue; }
if ( old2new[ismpl] != -1 ) { fprintf(bcftools_stderr,"Warning: The sample is listed multiple times: %s\n",ss); continue; }
- ss = se+1;
+ ss = se+(x != '\0');
while ( *ss && isspace(*ss) ) ss++;
if ( !*ss ) ss = "2"; // default ploidy
se = ss;
bcf_float_set_missing(args->missed_line->qual);
}
-static void print_missed_line(bcf_sr_regions_t *regs, void *data)
+static int tgt_parse(const char *line, char **chr_beg, char **chr_end, uint32_t *beg, uint32_t *end, void *payload, void *usr)
{
- args_t *args = (args_t*) data;
- call_t *call = &args->aux;
- bcf1_t *missed = args->missed_line;
+ char *ss = (char*) line;
+ while ( *ss && isspace(*ss) ) ss++;
+ if ( !*ss ) { fprintf(bcftools_stderr,"Could not parse the line: %s\n", line); return -2; }
+ if ( *ss=='#' ) return -1; // skip comments
- char *ss = regs->line.s;
- int i = 0;
- while ( i<args->aux.srs->targets_als-1 && *ss )
+ char *se = ss;
+ while ( *se && !isspace(*se) ) se++;
+
+ *chr_beg = ss;
+ *chr_end = se-1;
+
+ if ( !*se ) { fprintf(bcftools_stderr,"Could not parse the line: %s\n", line); return -2; }
+
+ ss = se+1;
+ *beg = strtod(ss, &se);
+ if ( ss==se ) { fprintf(bcftools_stderr,"Could not parse tab line: %s\n", line); return -2; }
+ if ( *beg==0 ) { fprintf(bcftools_stderr,"Could not parse tab line, expected 1-based coordinate: %s\n", line); return -2; }
+ (*beg)--;
+ *end = *beg;
+
+ if ( !usr ) return 0; // allele information not required
+
+ ss = se+1;
+ tgt_als_t *als = (tgt_als_t*)payload;
+ als->used = 0;
+ als->n = 0;
+ als->allele = NULL;
+ while ( *ss )
{
- if ( *ss=='\t' ) i++;
- ss++;
+ se = ss;
+ while ( *se && *se!=',' ) se++;
+ als->n++;
+ als->allele = (char**)realloc(als->allele,als->n*sizeof(*als->allele));
+ als->allele[als->n-1] = (char*)malloc(se-ss+1);
+ memcpy(als->allele[als->n-1],ss,se-ss);
+ als->allele[als->n-1][se-ss] = 0;
+ ss = se+1;
+ if ( !*se ) break;
}
- if ( !*ss ) error("Could not parse: [%s] (%d)\n", regs->line.s,args->aux.srs->targets_als);
+ return 0;
+}
+static void tgt_free(void *payload)
+{
+ tgt_als_t *als = (tgt_als_t*)payload;
+ int i;
+ for (i=0; i<als->n; i++) free(als->allele[i]);
+ free(als->allele);
+}
+static void tgt_flush_region(args_t *args, char *chr, uint32_t beg, uint32_t end)
+{
+ if ( !regidx_overlap(args->tgt_idx, chr,beg,end,args->tgt_itr_tmp) ) return;
+ while ( regitr_overlap(args->tgt_itr_tmp) )
+ {
+ if ( args->tgt_itr_tmp->beg < beg ) continue;
- missed->rid = bcf_hdr_name2id(call->hdr,regs->seq_names[regs->prev_seq]);
- missed->pos = regs->start;
- bcf_update_alleles_str(call->hdr, missed,ss);
+ tgt_als_t *tgt_als = ®itr_payload(args->tgt_itr_tmp,tgt_als_t);
+ if ( tgt_als->used ) continue;
- bcf_write1(args->out_fh, call->hdr, missed);
+ args->missed_line->rid = bcf_hdr_name2id(args->aux.hdr,chr);
+ args->missed_line->pos = args->tgt_itr_tmp->beg;
+ bcf_unpack(args->missed_line,BCF_UN_ALL);
+ bcf_update_alleles(args->aux.hdr, args->missed_line, (const char**)tgt_als->allele, tgt_als->n);
+ tgt_als->used = 1;
+ if ( bcf_write1(args->out_fh, args->aux.hdr, args->missed_line)!=0 ) error("[%s] Error: failed to write to %s\n", __func__,args->output_fname);
+ }
+}
+static void tgt_flush(args_t *args, bcf1_t *rec)
+{
+ if ( rec )
+ {
+ char *chr = (char*)bcf_seqname(args->aux.hdr,rec);
+
+ if ( !args->tgt_itr_prev ) // first record
+ tgt_flush_region(args,chr,0,rec->pos-1);
+
+ else if ( strcmp(chr,args->tgt_itr_prev->seq) ) // first record on a new chromosome
+ {
+ tgt_flush_region(args,args->tgt_itr_prev->seq,args->tgt_itr_prev->beg+1,REGIDX_MAX);
+ tgt_flush_region(args,chr,0,rec->pos-1);
+ }
+ else // another record on the same chromosome
+ tgt_flush_region(args,args->tgt_itr_prev->seq,args->tgt_itr_prev->beg,rec->pos-1);
+ }
+ else
+ {
+ // flush everything
+ if ( args->tgt_itr_prev )
+ tgt_flush_region(args,args->tgt_itr_prev->seq,args->tgt_itr_prev->beg,REGIDX_MAX);
+
+ int i, nchr = 0;
+ char **chr = regidx_seq_names(args->tgt_idx, &nchr);
+ for (i=0; i<nchr; i++)
+ tgt_flush_region(args,chr[i],0,REGIDX_MAX);
+ }
+}
+inline static int is_indel(int nals, char **als)
+{
+ // This is mpileup output, we can make some assumption:
+ // - no MNPs
+ // - "<*>" is not present at indels sites and there are no other symbolic alleles than <*>
+ if ( als[1][0]=='<' ) return 0;
+
+ int i;
+ for (i=0; i<nals; i++)
+ {
+ if ( als[i][0]=='<' ) continue;
+ if ( als[i][1] ) return 1;
+ }
+ return 0;
+}
+bcf1_t *next_line(args_t *args)
+{
+ bcf1_t *rec = NULL;
+ if ( !args->vcfbuf )
+ {
+ while ( bcf_sr_next_line(args->aux.srs) )
+ {
+ rec = args->aux.srs->readers[0].buffer[0];
+ if ( args->aux.srs->errnum || rec->errcode ) error("Error: could not parse the input VCF\n");
+ if ( args->tgt_idx )
+ {
+ if ( !regidx_overlap(args->tgt_idx, bcf_seqname(args->aux.hdr,rec),rec->pos,rec->pos,args->tgt_itr) ) continue;
+
+ // For backward compatibility: require the exact position, not an interval overlap
+ int pos_match = 0;
+ while ( regitr_overlap(args->tgt_itr) )
+ {
+ if ( args->tgt_itr->beg != rec->pos ) continue;
+ pos_match = 1;
+ break;
+ }
+ if ( !pos_match ) continue;
+ }
+ if ( args->samples_map ) bcf_subset(args->aux.hdr, rec, args->nsamples, args->samples_map);
+ bcf_unpack(rec, BCF_UN_STR);
+ return rec;
+ }
+ return NULL;
+ }
+
+ // If we are here,-C alleles was given and vcfbuf and tgt_idx are set
+
+ // Fill the buffer with duplicate lines
+ int vcfbuf_full = 1;
+ int nbuf = vcfbuf_nsites(args->vcfbuf);
+ bcf1_t *rec0 = NULL, *recN = NULL;
+ if ( nbuf==0 ) vcfbuf_full = 0;
+ else if ( nbuf==1 )
+ {
+ vcfbuf_full = 0;
+ rec0 = vcfbuf_peek(args->vcfbuf, 0);
+ }
+ else
+ {
+ rec0 = vcfbuf_peek(args->vcfbuf, 0);
+ recN = vcfbuf_peek(args->vcfbuf, nbuf-1);
+ if ( rec0->rid == recN->rid && rec0->pos == recN->pos ) vcfbuf_full = 0;
+ }
+ if ( !vcfbuf_full )
+ {
+ while ( bcf_sr_next_line(args->aux.srs) )
+ {
+ rec = args->aux.srs->readers[0].buffer[0];
+ if ( args->aux.srs->errnum || rec->errcode ) error("Error: could not parse the input VCF\n");
+ if ( !regidx_overlap(args->tgt_idx, bcf_seqname(args->aux.hdr,rec),rec->pos,rec->pos,args->tgt_itr) ) continue;
+ // as above: require the exact position, not an interval overlap
+ int exact_match = 0;
+ while ( regitr_overlap(args->tgt_itr) )
+ {
+ if ( args->tgt_itr->beg != rec->pos ) continue;
+ exact_match = 1;
+ break;
+ }
+ if ( !exact_match ) continue;
+
+ if ( args->samples_map ) bcf_subset(args->aux.hdr, rec, args->nsamples, args->samples_map);
+ bcf_unpack(rec, BCF_UN_STR);
+ if ( !rec0 ) rec0 = rec;
+ recN = rec;
+ args->aux.srs->readers[0].buffer[0] = vcfbuf_push(args->vcfbuf, rec, 1);
+ if ( rec0->rid!=recN->rid || rec0->pos!=recN->pos ) break;
+ }
+ }
+
+ nbuf = vcfbuf_nsites(args->vcfbuf);
+ int n, i,j;
+ for (n=nbuf; n>1; n--)
+ {
+ recN = vcfbuf_peek(args->vcfbuf, n-1);
+ if ( rec0->rid==recN->rid && rec0->pos==recN->pos ) break;
+ }
+ if ( n==0 )
+ {
+ assert( !nbuf );
+ return NULL;
+ }
+
+ // Find the VCF and tab record with the best matching combination of alleles, prioritize
+ // records of the same type (snp vs indel)
+ rec_tgt_t rec_tgt;
+ memset(&rec_tgt,0,sizeof(rec_tgt));
+ regidx_overlap(args->tgt_idx, bcf_seqname(args->aux.hdr,rec0),rec0->pos,rec0->pos,args->tgt_itr);
+ regitr_t *tmp_itr = regitr_init(args->tgt_idx);
+ regitr_copy(tmp_itr, args->tgt_itr);
+ for (i=0; i<n; i++)
+ {
+ rec = vcfbuf_peek(args->vcfbuf, i);
+ int rec_indel = is_indel(rec->n_allele, rec->d.allele) ? 1 : -1;
+ while ( regitr_overlap(tmp_itr) )
+ {
+ if ( tmp_itr->beg != rec->pos ) continue;
+ tgt_als_t *als = ®itr_payload(tmp_itr,tgt_als_t);
+ if ( als->used ) continue;
+ int nmatch_als = 0;
+ vcmp_t *vcmp = vcmp_init();
+ int ret = vcmp_set_ref(vcmp, rec->d.allele[0], als->allele[0]);
+ if ( ret==0 )
+ {
+ nmatch_als++;
+ if ( rec->n_allele > 1 && als->n > 1 )
+ {
+ for (j=1; j<als->n; j++)
+ {
+ if ( vcmp_find_allele(vcmp, rec->d.allele+1, rec->n_allele-1, als->allele[j])>=0 ) nmatch_als++;
+ }
+ }
+ }
+ int als_indel = is_indel(als->n, als->allele) ? 1 : -1;
+ nmatch_als *= rec_indel*als_indel;
+ if ( nmatch_als > rec_tgt.nmatch_als || !rec_tgt.als )
+ {
+ rec_tgt.nmatch_als = nmatch_als;
+ rec_tgt.als = als;
+ rec_tgt.ibuf = i;
+ }
+ vcmp_destroy(vcmp);
+ }
+ }
+ regitr_destroy(tmp_itr);
+
+ args->aux.tgt_als = rec_tgt.als;
+ if ( rec_tgt.als ) rec_tgt.als->used = 1;
+
+ rec = vcfbuf_remove(args->vcfbuf, rec_tgt.ibuf);
+ return rec;
}
static void init_data(args_t *args)
// Open files for input and output, initialize structures
if ( args->targets )
{
- if ( bcf_sr_set_targets(args->aux.srs, args->targets, args->targets_is_file, args->aux.flag&CALL_CONSTR_ALLELES ? 3 : 0)<0 )
- error("Failed to read the targets: %s\n", args->targets);
-
- if ( args->aux.flag&CALL_CONSTR_ALLELES && args->flag&CF_INS_MISSED )
- {
- args->aux.srs->targets->missed_reg_handler = print_missed_line;
- args->aux.srs->targets->missed_reg_data = args;
- }
+ args->tgt_idx = regidx_init(args->targets, tgt_parse, args->aux.flag&CALL_CONSTR_ALLELES ? tgt_free : NULL, sizeof(tgt_als_t), args->aux.flag&CALL_CONSTR_ALLELES ? args : NULL);
+ args->tgt_itr = regitr_init(args->tgt_idx);
+ args->tgt_itr_tmp = regitr_init(args->tgt_idx);
}
+
if ( args->regions )
{
if ( bcf_sr_set_regions(args->aux.srs, args->regions, args->regions_is_file)<0 )
error("Failed to read the regions: %s\n", args->regions);
}
- if ( !bcf_sr_add_reader(args->aux.srs, args->bcf_fname) ) error("Failed to open %s: %s\n", args->bcf_fname,bcf_sr_strerror(args->aux.srs->errnum));
+ if ( !bcf_sr_add_reader(args->aux.srs, args->bcf_fname) )
+ error("Failed to read from %s: %s\n", !strcmp("-",args->bcf_fname)?"standard input":args->bcf_fname,bcf_sr_strerror(args->aux.srs->errnum));
args->aux.hdr = bcf_sr_get_header(args->aux.srs,0);
int i;
}
}
+ if ( args->aux.flag & CALL_CONSTR_ALLELES )
+ args->vcfbuf = vcfbuf_init(args->aux.hdr, 0);
+
args->out_fh = hts_open(args->output_fname, hts_bcf_wmode(args->output_type));
- if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
+ if ( args->out_fh == NULL ) error("Error: cannot write to \"%s\": %s\n", args->output_fname, strerror(errno));
if ( args->n_threads ) hts_set_threads(args->out_fh, args->n_threads);
if ( args->flag & CF_QCALL )
bcf_hdr_remove(args->aux.hdr, BCF_HL_INFO, "I16");
if (args->record_cmd_line) bcf_hdr_append_version(args->aux.hdr, args->argc, args->argv, "bcftools_call");
- bcf_hdr_write(args->out_fh, args->aux.hdr);
+ if ( bcf_hdr_write(args->out_fh, args->aux.hdr)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname);
if ( args->flag&CF_INS_MISSED ) init_missed_line(args);
}
static void destroy_data(args_t *args)
{
+ if ( args->vcfbuf ) vcfbuf_destroy(args->vcfbuf);
+ if ( args->tgt_idx )
+ {
+ regidx_destroy(args->tgt_idx);
+ regitr_destroy(args->tgt_itr);
+ regitr_destroy(args->tgt_itr_tmp);
+ if ( args->tgt_itr_prev ) regitr_destroy(args->tgt_itr_prev);
+ }
if ( args->flag & CF_CCALL ) ccall_destroy(&args->aux);
else if ( args->flag & CF_MCALL ) mcall_destroy(&args->aux);
else if ( args->flag & CF_QCALL ) qcall_destroy(&args->aux);
free(args->samples_map);
free(args->sample2sex);
free(args->aux.ploidy);
+ free(args->str.s);
if ( args->gvcf ) gvcf_destroy(args->gvcf);
bcf_hdr_destroy(args->aux.hdr);
- hts_close(args->out_fh);
+ if ( hts_close(args->out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
bcf_sr_destroy(args->aux.srs);
}
static void usage(args_t *args)
{
fprintf(bcftools_stderr, "\n");
- fprintf(bcftools_stderr, "About: SNP/indel variant calling from VCF/BCF. To be used in conjunction with samtools mpileup.\n");
+ fprintf(bcftools_stderr, "About: SNP/indel variant calling from VCF/BCF. To be used in conjunction with bcftools mpileup.\n");
fprintf(bcftools_stderr, " This command replaces the former \"bcftools view\" caller. Some of the original\n");
fprintf(bcftools_stderr, " functionality has been temporarily lost in the process of transition to htslib,\n");
fprintf(bcftools_stderr, " but will be added back on popular demand. The original calling model can be\n");
fprintf(bcftools_stderr, " -S, --samples-file <file> PED file or a file with an optional column with sex (see man page for details) [all samples]\n");
fprintf(bcftools_stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(bcftools_stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(bcftools_stderr, "\n");
fprintf(bcftools_stderr, "Input/output options:\n");
fprintf(bcftools_stderr, " -A, --keep-alts keep all possible alternate alleles at variant sites\n");
fprintf(bcftools_stderr, " -f, --format-fields <list> output format fields: GQ,GP (lowercase allowed) []\n");
fprintf(bcftools_stderr, " -F, --prior-freqs <AN,AC> use prior allele frequencies\n");
+ fprintf(bcftools_stderr, " -G, --group-samples <file|-> group samples by population (file with \"sample\\tgroup\") or \"-\" for single-sample calling\n");
fprintf(bcftools_stderr, " -g, --gvcf <int>,[...] group non-variant sites into gVCF blocks by minimum per-sample DP\n");
fprintf(bcftools_stderr, " -i, --insert-missed output also sites missed by mpileup but present in -T\n");
fprintf(bcftools_stderr, " -M, --keep-masked-ref keep sites with masked reference allele (REF=N)\n");
fprintf(bcftools_stderr, " -n, --novel-rate <float>,[...] likelihood of novel mutation for constrained trio calling, see man page for details [1e-8,1e-9,1e-9]\n");
fprintf(bcftools_stderr, " -p, --pval-threshold <float> variant if P(ref|D)<FLOAT with -c [0.5]\n");
fprintf(bcftools_stderr, " -P, --prior <float> mutation rate (use bigger for greater sensitivity), use with -m [1.1e-3]\n");
+ fprintf(bcftools_stderr, "\n");
+ fprintf(bcftools_stderr, "Example:\n");
+ fprintf(bcftools_stderr, " # See also http://samtools.github.io/bcftools/howtos/variant-calling.html\n");
+ fprintf(bcftools_stderr, " bcftools mpileup -f reference.fa alignments.bam | bcftools call -mv -Ob -o calls.bcf\n");
// todo (and more)
// fprintf(bcftools_stderr, "\nContrast calling and association test options:\n");
{"format-fields",required_argument,NULL,'f'},
{"prior-freqs",required_argument,NULL,'F'},
{"gvcf",required_argument,NULL,'g'},
+ {"group-samples",required_argument,NULL,'G'},
{"output",required_argument,NULL,'o'},
{"output-type",required_argument,NULL,'O'},
{"regions",required_argument,NULL,'r'},
};
char *tmp = NULL;
- while ((c = getopt_long(argc, argv, "h?o:O:r:R:s:S:t:T:ANMV:vcmp:C:n:P:f:ig:XYF:", loptions, NULL)) >= 0)
+ while ((c = getopt_long(argc, argv, "h?o:O:r:R:s:S:t:T:ANMV:vcmp:C:n:P:f:ig:XYF:G:", loptions, NULL)) >= 0)
{
switch (c)
{
case 1 : ploidy = optarg; break;
case 'X': ploidy = "X"; fprintf(bcftools_stderr,"Warning: -X will be deprecated, please use --ploidy instead.\n"); break;
case 'Y': ploidy = "Y"; fprintf(bcftools_stderr,"Warning: -Y will be deprecated, please use --ploidy instead.\n"); break;
+ case 'G': args.aux.sample_groups = optarg; break;
case 'f': args.aux.output_tags |= parse_format_flag(optarg); break;
case 'M': args.flag &= ~CF_ACGT_ONLY; break; // keep sites where REF is N
case 'N': args.flag |= CF_ACGT_ONLY; break; // omit sites where first base in REF is N (the new default)
}
if ( args.flag & CF_INS_MISSED && !(args.aux.flag&CALL_CONSTR_ALLELES) ) error("The -i option requires -C alleles\n");
if ( args.aux.flag&CALL_VARONLY && args.gvcf ) error("The two options cannot be combined: --variants-only and --gvcf\n");
+ if ( args.aux.sample_groups && !(args.flag & CF_MCALL) ) error("The -G feature is supported only with the -m calling mode\n");
init_data(&args);
- while ( bcf_sr_next_line(args.aux.srs) )
+ bcf1_t *bcf_rec;
+ while ( (bcf_rec = next_line(&args)) )
{
- bcf1_t *bcf_rec = args.aux.srs->readers[0].buffer[0];
- if ( args.samples_map ) bcf_subset(args.aux.hdr, bcf_rec, args.nsamples, args.samples_map);
- bcf_unpack(bcf_rec, BCF_UN_STR);
+ // Skip duplicate positions with all matching `-C alleles -T` used up
+ if ( args.aux.flag&CALL_CONSTR_ALLELES && !args.aux.tgt_als ) continue;
// Skip unwanted sites
int i, is_indel = bcf_is_snp(bcf_rec) ? 0 : 1;
continue;
}
+ if ( args.flag & CF_INS_MISSED )
+ {
+ tgt_flush(&args,bcf_rec);
+ if ( !args.tgt_itr_prev ) args.tgt_itr_prev = regitr_init(args.tgt_idx);
+ regitr_copy(args.tgt_itr_prev, args.tgt_itr);
+ }
+
// Calling modes which output VCFs
int ret;
if ( args.flag & CF_MCALL )
if ( (args.aux.flag & CALL_VARONLY) && ret==0 && !args.gvcf ) continue; // not a variant
if ( args.gvcf )
bcf_rec = gvcf_write(args.gvcf, args.out_fh, args.aux.hdr, bcf_rec, ret==1?1:0);
- if ( bcf_rec )
- bcf_write1(args.out_fh, args.aux.hdr, bcf_rec);
+ if ( bcf_rec && bcf_write1(args.out_fh, args.aux.hdr, bcf_rec)!=0 ) error("[%s] Error: failed to write to %s\n", __func__,args.output_fname);
}
if ( args.gvcf ) gvcf_write(args.gvcf, args.out_fh, args.aux.hdr, NULL, 0);
- if ( args.flag & CF_INS_MISSED ) bcf_sr_regions_flush(args.aux.srs->targets);
+ if ( args.flag & CF_INS_MISSED ) tgt_flush(&args,NULL);
destroy_data(&args);
return 0;
}
#include <unistd.h>
#include <getopt.h>
#include <math.h>
+#include <inttypes.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/kstring.h>
}
static void close_sample_files(sample_t *smpl)
{
- fclose(smpl->dat_fh);
- fclose(smpl->cn_fh);
- fclose(smpl->summary_fh);
+ if ( fclose(smpl->dat_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,smpl->dat_fname);
+ if ( fclose(smpl->cn_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,smpl->cn_fname);
+ if ( fclose(smpl->summary_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,smpl->summary_fname);
}
static double norm_cdf(double mean, double dev);
args->control_sample.lrr[args->nsites-1] = lrr2;
args->control_sample.baf[args->nsites-1] = baf2;
if ( baf2>=0 ) // skip missing values
- fprintf(args->control_sample.dat_fh,"%s\t%d\t%.3f\t%.3f\n",bcf_hdr_id2name(args->hdr,args->prev_rid), line->pos+1,baf2,lrr2);
+ fprintf(args->control_sample.dat_fh,"%s\t%"PRId64"\t%.3f\t%.3f\n",bcf_hdr_id2name(args->hdr,args->prev_rid), (int64_t) line->pos+1,baf2,lrr2);
}
if ( baf1>=0 ) // skip missing values
- fprintf(args->query_sample.dat_fh,"%s\t%d\t%.3f\t%.3f\n",bcf_hdr_id2name(args->hdr,args->prev_rid), line->pos+1,baf1,lrr1);
+ fprintf(args->query_sample.dat_fh,"%s\t%"PRId64"\t%.3f\t%.3f\n",bcf_hdr_id2name(args->hdr,args->prev_rid), (int64_t) line->pos+1,baf1,lrr1);
if ( baf1>=0 )
{
{"LRR-weight",1,0,'l'},
{"same-prob",1,0,'P'},
{"xy-prob",1,0,'x'},
- {"sample",1,0,'s'},
- {"control",1,0,'c'},
+ {"query-sample",1,0,'s'},
+ {"control-sample",1,0,'c'},
{"targets",1,0,'t'},
{"targets-file",1,0,'T'},
{"regions",1,0,'r'},
{"regions-file",1,0,'R'},
- {"plot",1,0,'p'},
+ {"plot-threshold",1,0,'p'},
{"output-dir",1,0,'o'},
{0,0,0,0}
};
if ( bcf_sr_set_targets(args->files, args->af_fname, 1, 3)<0 )
error("Failed to read the targets: %s\n", args->af_fname);
}
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) )
+ error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
init_data(args);
while ( bcf_sr_next_line(args->files) )
#include <unistd.h>
#include <getopt.h>
#include <math.h>
+#include <inttypes.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/kstring.h>
}
static void close_sample_files(sample_t *smpl)
{
- fclose(smpl->dat_fh);
- fclose(smpl->cn_fh);
- fclose(smpl->summary_fh);
+ if ( fclose(smpl->dat_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,smpl->dat_fname);
+ if ( fclose(smpl->cn_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,smpl->cn_fname);
+ if ( fclose(smpl->summary_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,smpl->summary_fname);
}
static double norm_cdf(double mean, double dev);
args->control_sample.lrr[args->nsites-1] = lrr2;
args->control_sample.baf[args->nsites-1] = baf2;
if ( baf2>=0 ) // skip missing values
- fprintf(args->control_sample.dat_fh,"%s\t%d\t%.3f\t%.3f\n",bcf_hdr_id2name(args->hdr,args->prev_rid), line->pos+1,baf2,lrr2);
+ fprintf(args->control_sample.dat_fh,"%s\t%"PRId64"\t%.3f\t%.3f\n",bcf_hdr_id2name(args->hdr,args->prev_rid), (int64_t) line->pos+1,baf2,lrr2);
}
if ( baf1>=0 ) // skip missing values
- fprintf(args->query_sample.dat_fh,"%s\t%d\t%.3f\t%.3f\n",bcf_hdr_id2name(args->hdr,args->prev_rid), line->pos+1,baf1,lrr1);
+ fprintf(args->query_sample.dat_fh,"%s\t%"PRId64"\t%.3f\t%.3f\n",bcf_hdr_id2name(args->hdr,args->prev_rid), (int64_t) line->pos+1,baf1,lrr1);
if ( baf1>=0 )
{
{"LRR-weight",1,0,'l'},
{"same-prob",1,0,'P'},
{"xy-prob",1,0,'x'},
- {"sample",1,0,'s'},
- {"control",1,0,'c'},
+ {"query-sample",1,0,'s'},
+ {"control-sample",1,0,'c'},
{"targets",1,0,'t'},
{"targets-file",1,0,'T'},
{"regions",1,0,'r'},
{"regions-file",1,0,'R'},
- {"plot",1,0,'p'},
+ {"plot-threshold",1,0,'p'},
{"output-dir",1,0,'o'},
{0,0,0,0}
};
if ( bcf_sr_set_targets(args->files, args->af_fname, 1, 3)<0 )
error("Failed to read the targets: %s\n", args->af_fname);
}
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) )
+ error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
init_data(args);
while ( bcf_sr_next_line(args->files) )
/* vcfconcat.c -- Concatenate or combine VCF/BCF files.
- Copyright (C) 2013-2015 Genome Research Ltd.
+ Copyright (C) 2013-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#include <htslib/kseq.h>
#include <htslib/bgzf.h>
#include <htslib/tbx.h> // for hts_get_bgzfp()
+#include <htslib/thread_pool.h>
+#include <sys/time.h>
#include "bcftools.h"
typedef struct _args_t
char **argv, *output_fname, *file_list, **fnames, *remove_dups, *regions_list;
int argc, nfnames, allow_overlaps, phased_concat, regions_is_file;
- int compact_PS, phase_set_changed, naive_concat;
+ int compact_PS, phase_set_changed, naive_concat, naive_concat_trust_headers;
+ int verbose;
+ htsThreadPool *tpool;
}
args_t;
line = bcf_init();
}
+ if ( args->verbose ) fprintf(stderr,"Checking the headers and starting positions of %d files\n", args->nfnames);
kstring_t str = {0,0,0};
int i, prev_chrid = -1;
for (i=0; i<args->nfnames; i++)
}
}
bcf_hdr_destroy(hdr);
- hts_close(fp);
+ if ( hts_close(fp)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fnames[i]);
}
free(str.s);
if ( line ) bcf_destroy(line);
if (args->record_cmd_line) bcf_hdr_append_version(args->out_hdr, args->argc, args->argv, "bcftools_concat");
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
- if ( args->n_threads ) hts_set_threads(args->out_fh, args->n_threads);
-
- bcf_hdr_write(args->out_fh, args->out_hdr);
-
- if ( args->allow_overlaps )
+ if ( args->allow_overlaps || args->phased_concat )
{
args->files = bcf_sr_init();
args->files->require_index = 1;
+ }
+ if ( args->n_threads )
+ {
+ if ( args->files )
+ {
+ if ( bcf_sr_set_threads(args->files, args->n_threads)<0 ) error("Failed to create threads\n");
+ args->tpool = args->files->p;
+ }
+ else
+ {
+ args->tpool = (htsThreadPool*) calloc(1, sizeof(htsThreadPool));
+ if ( !args->tpool ) error("Failed to allocate memory\n");
+ if ( !(args->tpool->pool = hts_tpool_init(args->n_threads)) ) error("Failed to initialize %d threads\n",args->n_threads);
+ }
+ hts_set_opt(args->out_fh, HTS_OPT_THREAD_POOL, args->tpool);
+ }
+ if ( bcf_hdr_write(args->out_fh, args->out_hdr)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname);
+
+ if ( args->allow_overlaps )
+ {
if ( args->regions_list )
{
if ( bcf_sr_set_regions(args->files, args->regions_list, args->regions_is_file)<0 )
args->nmism = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->phase_qual = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
args->phase_set = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
- args->files = bcf_sr_init();
- args->files->require_index = 1;
args->ifname = 0;
}
}
static void destroy_data(args_t *args)
{
int i;
- for (i=0; i<args->nfnames; i++) free(args->fnames[i]);
- free(args->fnames);
- if ( args->files ) bcf_sr_destroy(args->files);
if ( args->out_fh )
{
if ( hts_close(args->out_fh)!=0 ) error("hts_close error\n");
}
+ if ( args->tpool && !args->files )
+ {
+ hts_tpool_destroy(args->tpool->pool);
+ free(args->tpool);
+ }
+ if ( args->files ) bcf_sr_destroy(args->files);
if ( args->out_hdr ) bcf_hdr_destroy(args->out_hdr);
free(args->seen_seq);
free(args->start_pos);
free(args->nmism);
free(args->phase_qual);
free(args->phase_set);
+ for (i=0; i<args->nfnames; i++) free(args->fnames[i]);
+ free(args->fnames);
}
int vcf_write_line(htsFile *fp, kstring_t *line);
{
if ( !gt_absent_warned )
{
- fprintf(stderr,"GT is not present at %s:%d. (This warning is printed only once.)\n", bcf_seqname(ahdr,arec), arec->pos+1);
+ fprintf(stderr,"GT is not present at %s:%"PRId64". (This warning is printed only once.)\n", bcf_seqname(ahdr,arec), (int64_t) arec->pos+1);
gt_absent_warned = 1;
}
continue;
{
if ( !gt_absent_warned )
{
- fprintf(stderr,"GT is not present at %s:%d. (This warning is printed only once.)\n", bcf_seqname(bhdr,brec), brec->pos+1);
+ fprintf(stderr,"GT is not present at %s:%"PRId64". (This warning is printed only once.)\n", bcf_seqname(bhdr,brec), (int64_t) brec->pos+1);
gt_absent_warned = 1;
}
continue;
bcf_update_format_int32(args->out_hdr,arec,"PS",args->phase_set,nsmpl);
args->phase_set_changed = 0;
}
- bcf_write(args->out_fh, args->out_hdr, arec);
+ if ( bcf_write(args->out_fh, args->out_hdr, arec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
- if ( arec->pos < args->prev_pos_check ) error("FIXME, disorder: %s:%d vs %d [1]\n", bcf_seqname(args->files->readers[0].header,arec),arec->pos+1,args->prev_pos_check+1);
+ if ( arec->pos < args->prev_pos_check ) error("FIXME, disorder: %s:%"PRId64" vs %d [1]\n", bcf_seqname(args->files->readers[0].header,arec),(int64_t) arec->pos+1,args->prev_pos_check+1);
args->prev_pos_check = arec->pos;
}
args->nswap = 0;
bcf_update_format_int32(args->out_hdr,brec,"PS",args->phase_set,nsmpl);
args->phase_set_changed = 0;
}
- bcf_write(args->out_fh, args->out_hdr, brec);
+ if ( bcf_write(args->out_fh, args->out_hdr, brec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
- if ( brec->pos < args->prev_pos_check ) error("FIXME, disorder: %s:%d vs %d [2]\n", bcf_seqname(args->files->readers[1].header,brec),brec->pos+1,args->prev_pos_check+1);
+ if ( brec->pos < args->prev_pos_check ) error("FIXME, disorder: %s:%"PRId64" vs %d [2]\n", bcf_seqname(args->files->readers[1].header,brec),(int64_t) brec->pos+1,args->prev_pos_check+1);
args->prev_pos_check = brec->pos;
}
args->nbuf = 0;
static void phased_push(args_t *args, bcf1_t *arec, bcf1_t *brec)
{
if ( arec && arec->errcode )
- error("Parse error at %s:%d, cannot proceed: %s\n", bcf_seqname(args->files->readers[0].header,arec),arec->pos+1, args->files->readers[0].fname);
+ error("Parse error at %s:%"PRId64", cannot proceed: %s\n", bcf_seqname(args->files->readers[0].header,arec),(int64_t) arec->pos+1, args->files->readers[0].fname);
if ( brec && brec->errcode )
- error("Parse error at %s:%d, cannot proceed: %s\n", bcf_seqname(args->files->readers[1].header,brec),brec->pos+1, args->files->readers[1].fname);
+ error("Parse error at %s:%"PRId64", cannot proceed: %s\n", bcf_seqname(args->files->readers[1].header,brec),(int64_t) brec->pos+1, args->files->readers[1].fname);
int i, nsmpl = bcf_hdr_nsamples(args->out_hdr);
int chr_id = bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[0].header,arec));
bcf_update_format_int32(args->out_hdr,arec,"PS",args->phase_set,nsmpl);
args->phase_set_changed = 0;
}
- bcf_write(args->out_fh, args->out_hdr, arec);
+ if ( bcf_write(args->out_fh, args->out_hdr, arec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
if ( arec->pos < args->prev_pos_check )
- error("FIXME, disorder: %s:%d in %s vs %d written [3]\n", bcf_seqname(args->files->readers[0].header,arec), arec->pos+1,args->files->readers[0].fname, args->prev_pos_check+1);
+ error("FIXME, disorder: %s:%"PRId64" in %s vs %d written [3]\n", bcf_seqname(args->files->readers[0].header,arec), (int64_t) arec->pos+1,args->files->readers[0].fname, args->prev_pos_check+1);
args->prev_pos_check = arec->pos;
return;
}
static void concat(args_t *args)
{
+ static int site_drop_warned = 0;
int i;
if ( args->phased_concat ) // phased concat
{
if ( !bcf_sr_has_line(args->files,0) ) // no input from the first reader
{
// We are assuming that there is a perfect overlap, sites which are not present in both files are dropped
- if ( ! bcf_sr_region_done(args->files,0) ) continue;
-
+ if ( ! bcf_sr_region_done(args->files,0) )
+ {
+ if ( !site_drop_warned )
+ {
+ fprintf(stderr,
+ "Warning: Dropping the site %s:%"PRId64". The --ligate option is intended for VCFs with perfect\n"
+ " overlap, sites in overlapping regions present in one but missing in other are dropped.\n"
+ " This warning is printed only once.\n",
+ bcf_seqname(bcf_sr_get_header(args->files,1),bcf_sr_get_line(args->files,1)), (int64_t) bcf_sr_get_line(args->files,1)->pos+1
+ );
+ site_drop_warned = 1;
+ }
+ continue;
+ }
phased_flush(args);
bcf_sr_remove_reader(args->files, 0);
}
bcf1_t *line = bcf_sr_get_line(args->files,i);
if ( !line ) continue;
bcf_translate(args->out_hdr, args->files->readers[i].header, line);
- bcf_write1(args->out_fh, args->out_hdr, line);
+ if ( bcf_write1(args->out_fh, args->out_hdr, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
if ( args->remove_dups ) break;
}
}
}
else // concatenating
{
+ struct timeval t0, t1;
kstring_t tmp = {0,0,0};
int prev_chr_id = -1, prev_pos;
bcf1_t *line = bcf_init();
for (i=0; i<args->nfnames; i++)
{
- htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
- bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
+ if ( args->verbose )
+ {
+ fprintf(stderr,"Concatenating %s", args->fnames[i]);
+ gettimeofday(&t0, NULL);
+ }
+ htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("\nFailed to open: %s\n", args->fnames[i]);
+ if ( args->n_threads ) hts_set_opt(fp, HTS_OPT_THREAD_POOL, args->tpool);
+ bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("\nFailed to parse header: %s\n", args->fnames[i]);
if ( !fp->is_bin && args->output_type&FT_VCF )
{
line->max_unpack = BCF_UN_STR;
tmp.l = 0;
kputsn(fp->line.s,str-fp->line.s,&tmp);
int chr_id = bcf_hdr_name2id(args->out_hdr, tmp.s);
- if ( chr_id<0 ) error("The sequence \"%s\" not defined in the header: %s\n(Quick workaround: index the file.)\n", tmp.s, args->fnames[i]);
+ if ( chr_id<0 ) error("\nThe sequence \"%s\" not defined in the header: %s\n(Quick workaround: index the file.)\n", tmp.s, args->fnames[i]);
if ( prev_chr_id!=chr_id )
{
prev_pos = -1;
int pos = strtol(str+1,&end,10) - 1;
if ( end==str+1 ) error("Could not parse line: %s\n", fp->line.s);
if ( prev_pos > pos )
- error("The chromosome block %s is not sorted, consider running with -a.\n", tmp.s);
+ error("\nThe chromosome block %s is not sorted, consider running with -a.\n", tmp.s);
args->seen_seq[chr_id] = 1;
prev_chr_id = chr_id;
- if ( vcf_write_line(args->out_fh, &fp->line)!=0 ) error("Failed to write %"PRIu64" bytes\n", (uint64_t)fp->line.l);
+ if ( vcf_write_line(args->out_fh, &fp->line)!=0 ) error("\nFailed to write %"PRIu64" bytes\n", (uint64_t)fp->line.l);
}
}
else
error("\nThe chromosome block %s is not contiguous, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
}
if ( prev_pos > line->pos )
- error("The chromosome block %s is not sorted, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
+ error("\nThe chromosome block %s is not sorted, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
args->seen_seq[line->rid] = 1;
prev_chr_id = line->rid;
- if ( bcf_write(args->out_fh, args->out_hdr, line)!=0 ) error("Failed to write\n");
+ if ( bcf_write(args->out_fh, args->out_hdr, line)!=0 ) error("\nFailed to write\n");
}
}
bcf_hdr_destroy(hdr);
hts_close(fp);
+ if ( args->verbose )
+ {
+ gettimeofday(&t1, NULL);
+ double delta = (t1.tv_sec - t0.tv_sec) * 1e6 + (t1.tv_usec - t0.tv_usec);
+ fprintf(stderr,"\t%f seconds\n",delta/1e6);
+ }
}
bcf_destroy(line);
free(tmp.s);
&& header[12] == 'B' && header[13] == 'C'
&& unpackInt16((uint8_t*)&header[14]) == 2) ? 0 : -1;
}
+static void _check_hrecs(const bcf_hdr_t *hdr0, const bcf_hdr_t *hdr, char *fname0, char *fname)
+{
+ int j;
+ for (j=0; j<hdr0->nhrec; j++)
+ {
+ bcf_hrec_t *hrec0 = hdr0->hrec[j];
+ if ( hrec0->type!=BCF_HL_FLT && hrec0->type!=BCF_HL_INFO && hrec0->type!=BCF_HL_FMT && hrec0->type!=BCF_HL_CTG ) continue; // skip fiels w/o IDX
+ int itag = bcf_hrec_find_key(hrec0, "ID");
+ bcf_hrec_t *hrec = bcf_hdr_get_hrec(hdr, hrec0->type, "ID", hrec0->vals[itag], NULL);
+
+ char *type = NULL;
+ if ( hrec0->type==BCF_HL_FLT ) type = "FILTER";
+ if ( hrec0->type==BCF_HL_INFO ) type = "INFO";
+ if ( hrec0->type==BCF_HL_FMT ) type = "FORMAT";
+ if ( hrec0->type==BCF_HL_CTG ) type = "contig";
+
+ if ( !hrec )
+ error("Cannot use --naive, incompatible headers, the tag %s/%s not present in %s\n",type,hrec0->vals[itag],fname);
+
+ int idx0 = bcf_hrec_find_key(hrec0, "IDX");
+ int idx = bcf_hrec_find_key(hrec, "IDX");
+ if ( idx0<0 || idx<0 )
+ error("fixme: unexpected IDX<0 for %s/%s in %s or %s\n",type,hrec0->vals[itag],fname0,fname);
+ if ( strcmp(hrec0->vals[idx0],hrec->vals[idx]) )
+ error("Cannot use --naive, use --naive-force instead: different order the tag %s/%s in %s vs %s\n",type,hrec0->vals[itag],fname0,fname);
+ }
+}
+static void naive_concat_check_headers(args_t *args)
+{
+ fprintf(stderr,"Checking the headers of %d files.\n",args->nfnames);
+ bcf_hdr_t *hdr0 = NULL;
+ int i,j;
+ for (i=0; i<args->nfnames; i++)
+ {
+ htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
+ bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
+ htsFormat type = *hts_get_format(fp);
+ hts_close(fp);
+
+ if ( i==0 )
+ {
+ hdr0 = hdr;
+ continue;
+ }
+
+ // check the samples
+ if ( bcf_hdr_nsamples(hdr0)!=bcf_hdr_nsamples(hdr) )
+ error("Cannot concatenate, different number of samples: %d vs %d in %s vs %s\n",bcf_hdr_nsamples(hdr0),bcf_hdr_nsamples(hdr),args->fnames[0],args->fnames[i]);
+ for (j=0; j<bcf_hdr_nsamples(hdr0); j++)
+ if ( strcmp(hdr0->samples[j],hdr->samples[j]) )
+ error("Cannot concatenate, different samples in %s vs %s\n",args->fnames[0],args->fnames[i]);
+
+ // if BCF, check if tag IDs are consistent in the dictionary of strings
+ if ( type.compression!=bgzf )
+ error("The --naive option works only for compressed BCFs or VCFs, sorry :-/\n");
+ if ( type.format==vcf )
+ {
+ bcf_hdr_destroy(hdr);
+ continue;
+ }
+
+ _check_hrecs(hdr0,hdr,args->fnames[0],args->fnames[i]);
+ _check_hrecs(hdr,hdr0,args->fnames[i],args->fnames[0]);
+
+ bcf_hdr_destroy(hdr);
+ }
+ if ( hdr0 ) bcf_hdr_destroy(hdr0);
+ fprintf(stderr,"Done, the headers are compatible.\n");
+}
static void naive_concat(args_t *args)
{
+ if ( !args->naive_concat_trust_headers )
+ naive_concat_check_headers(args);
+
// only compressed BCF atm
BGZF *bgzf_out = bgzf_open(args->output_fname,"w");;
+ struct timeval t0, t1;
const size_t page_size = BGZF_MAX_BLOCK_SIZE;
uint8_t *buf = (uint8_t*) malloc(page_size);
kstring_t tmp = {0,0,0};
int i, file_types = 0;
for (i=0; i<args->nfnames; i++)
{
+ if ( args->verbose )
+ {
+ fprintf(stderr,"Concatenating %s", args->fnames[i]);
+ gettimeofday(&t0, NULL);
+ }
htsFile *hts_fp = hts_open(args->fnames[i],"r");
- if ( !hts_fp ) error("Failed to open: %s\n", args->fnames[i]);
+ if ( !hts_fp ) error("\nFailed to open: %s\n", args->fnames[i]);
htsFormat type = *hts_get_format(hts_fp);
if ( type.compression!=bgzf )
- error("The --naive option works only for compressed BCFs or VCFs, sorry :-/\n");
+ error("\nThe --naive option works only for compressed BCFs or VCFs, sorry :-/\n");
file_types |= type.format==vcf ? 1 : 2;
if ( file_types==3 )
- error("The --naive option works only for compressed files of the same type, all BCFs or all VCFs :-/\n");
+ error("\nThe --naive option works only for compressed files of the same type, all BCFs or all VCFs :-/\n");
BGZF *fp = hts_get_bgzfp(hts_fp);
if ( !fp || bgzf_read_block(fp) != 0 || !fp->block_length )
- error("Failed to read %s: %s\n", args->fnames[i], strerror(errno));
+ error("\nFailed to read %s: %s\n", args->fnames[i], strerror(errno));
int nskip;
if ( type.format==bcf )
{
uint8_t magic[5];
- if ( bgzf_read(fp, magic, 5) != 5 ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
- if (strncmp((char*)magic, "BCF\2\2", 5) != 0) error("Invalid BCF magic string in %s\n", args->fnames[i]);
+ if ( bgzf_read(fp, magic, 5) != 5 ) error("\nFailed to read the BCF header in %s\n", args->fnames[i]);
+ if (strncmp((char*)magic, "BCF\2\2", 5) != 0) error("\nInvalid BCF magic string in %s\n", args->fnames[i]);
- if ( bgzf_read(fp, &tmp.l, 4) != 4 ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
+ if ( bgzf_read(fp, &tmp.l, 4) != 4 ) error("\nFailed to read the BCF header in %s\n", args->fnames[i]);
hts_expand(char,tmp.l,tmp.m,tmp.s);
- if ( bgzf_read(fp, tmp.s, tmp.l) != tmp.l ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
+ if ( bgzf_read(fp, tmp.s, tmp.l) != tmp.l ) error("\nFailed to read the BCF header in %s\n", args->fnames[i]);
// write only the first header
if ( i==0 )
{
- if ( bgzf_write(bgzf_out, "BCF\2\2", 5) !=5 ) error("Failed to write %d bytes to %s\n", 5,args->output_fname);
- if ( bgzf_write(bgzf_out, &tmp.l, 4) !=4 ) error("Failed to write %d bytes to %s\n", 4,args->output_fname);
- if ( bgzf_write(bgzf_out, tmp.s, tmp.l) != tmp.l) error("Failed to write %"PRId64" bytes to %s\n", (uint64_t)tmp.l,args->output_fname);
+ if ( bgzf_write(bgzf_out, "BCF\2\2", 5) !=5 ) error("\nFailed to write %d bytes to %s\n", 5,args->output_fname);
+ if ( bgzf_write(bgzf_out, &tmp.l, 4) !=4 ) error("\nFailed to write %d bytes to %s\n", 4,args->output_fname);
+ if ( bgzf_write(bgzf_out, tmp.s, tmp.l) != tmp.l) error("\nFailed to write %"PRId64" bytes to %s\n", (uint64_t)tmp.l,args->output_fname);
}
nskip = fp->block_offset;
}
else
{
nskip = print_vcf_gz_header(fp, bgzf_out, i==0?1:0, &tmp);
- if ( nskip==-1 ) error("Error reading %s\n", args->fnames[i]);
+ if ( nskip==-1 ) error("\nError reading %s\n", args->fnames[i]);
}
// Output all non-header data that were read together with the header block
if ( fp->block_length - nskip > 0 )
{
- if ( bgzf_write(bgzf_out, (char *)fp->uncompressed_block+nskip, fp->block_length-nskip)<0 ) error("Error: %d\n",fp->errcode);
+ if ( bgzf_write(bgzf_out, (char *)fp->uncompressed_block+nskip, fp->block_length-nskip)<0 ) error("\nError: %d\n",fp->errcode);
}
- if ( bgzf_flush(bgzf_out)<0 ) error("Error: %d\n",bgzf_out->errcode);
+ if ( bgzf_flush(bgzf_out)<0 ) error("\nError: %d\n",bgzf_out->errcode);
// Stream the rest of the file as it is, without recompressing, but remove BGZF EOF blocks
{
nread = bgzf_raw_read(fp, buf, nheader);
if ( !nread ) break;
- if ( nread != nheader || check_header(buf)!=0 ) error("Could not parse the header of a bgzf block: %s\n",args->fnames[i]);
+ if ( nread != nheader || check_header(buf)!=0 ) error("\nCould not parse the header of a bgzf block: %s\n",args->fnames[i]);
nblock = unpackInt16(buf+16) + 1;
assert( nblock <= page_size && nblock >= nheader );
nread += bgzf_raw_read(fp, buf+nheader, nblock - nheader);
- if ( nread!=nblock ) error("Could not read %"PRId64" bytes: %s\n",(uint64_t)nblock,args->fnames[i]);
+ if ( nread!=nblock ) error("\nCould not read %"PRId64" bytes: %s\n",(uint64_t)nblock,args->fnames[i]);
if ( nread==neof && !memcmp(buf,eof,neof) ) continue;
nwr = bgzf_raw_write(bgzf_out, buf, nread);
- if ( nwr != nread ) error("Write failed, wrote %"PRId64" instead of %d bytes.\n", (uint64_t)nwr,(int)nread);
+ if ( nwr != nread ) error("\nWrite failed, wrote %"PRId64" instead of %d bytes.\n", (uint64_t)nwr,(int)nread);
+ }
+ if (hts_close(hts_fp)) error("\nClose failed: %s\n",args->fnames[i]);
+ if ( args->verbose )
+ {
+ gettimeofday(&t1, NULL);
+ double delta = (t1.tv_sec - t0.tv_sec) * 1e6 + (t1.tv_usec - t0.tv_usec);
+ fprintf(stderr,"\t%f seconds\n",delta/1e6);
}
- if (hts_close(hts_fp)) error("Close failed: %s\n",args->fnames[i]);
}
free(buf);
free(tmp.s);
fprintf(stderr, " VCF into one. The input files must be sorted by chr and position. The files\n");
fprintf(stderr, " must be given in the correct order to produce sorted VCF on output unless\n");
fprintf(stderr, " the -a, --allow-overlaps option is specified. With the --naive option, the files\n");
- fprintf(stderr, " are concatenated without being recompressed, which is very fast but dangerous\n");
- fprintf(stderr, " if the BCF headers differ.\n");
+ fprintf(stderr, " are concatenated without being recompressed, which is very fast.\n");
fprintf(stderr, "Usage: bcftools concat [options] <A.vcf.gz> [<B.vcf.gz> [...]]\n");
fprintf(stderr, "\n");
fprintf(stderr, "Options:\n");
fprintf(stderr, " -f, --file-list <file> Read the list of files from a file.\n");
fprintf(stderr, " -l, --ligate Ligate phased VCFs by matching phase at overlapping haplotypes\n");
fprintf(stderr, " --no-version Do not append version and command line to the header\n");
- fprintf(stderr, " -n, --naive Concatenate files without recompression (dangerous, use with caution)\n");
+ fprintf(stderr, " -n, --naive Concatenate files without recompression, a header check compatibility is performed\n");
+ fprintf(stderr, " --naive-force Same as --naive, but header compatibility is not checked. Dangerous, use with caution.\n");
fprintf(stderr, " -o, --output <file> Write output to a file [standard output]\n");
fprintf(stderr, " -O, --output-type <b|u|z|v> b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n");
fprintf(stderr, " -q, --min-PQ <int> Break phase set if phasing quality is lower than <int> [30]\n");
fprintf(stderr, " -r, --regions <region> Restrict to comma-separated list of regions\n");
fprintf(stderr, " -R, --regions-file <file> Restrict to regions listed in a file\n");
- fprintf(stderr, " --threads <int> Number of extra output compression threads [0]\n");
+ fprintf(stderr, " --threads <int> Use multithreading with <int> worker threads [0]\n");
+ fprintf(stderr, " -v, --verbose <0|1> Set verbosity level [1]\n");
fprintf(stderr, "\n");
exit(1);
}
args->n_threads = 0;
args->record_cmd_line = 1;
args->min_PQ = 30;
+ args->verbose = 1;
static struct option loptions[] =
{
+ {"verbose",required_argument,NULL,'v'},
{"naive",no_argument,NULL,'n'},
+ {"naive-force",no_argument,NULL,7},
{"compact-PS",no_argument,NULL,'c'},
{"regions",required_argument,NULL,'r'},
{"regions-file",required_argument,NULL,'R'},
{NULL,0,NULL,0}
};
char *tmp;
- while ((c = getopt_long(argc, argv, "h:?o:O:f:alq:Dd:r:R:cn",loptions,NULL)) >= 0)
+ while ((c = getopt_long(argc, argv, "h:?o:O:f:alq:Dd:r:R:cnv:",loptions,NULL)) >= 0)
{
switch (c) {
case 'c': args->compact_PS = 1; break;
break;
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
+ case 7 : args->naive_concat = 1; args->naive_concat_trust_headers = 1; break;
+ case 'v':
+ args->verbose = strtol(optarg, 0, 0);
+ error("Error: currently only --verbose 0 or --verbose 1 is supported\n");
+ break;
case 'h':
case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
args->fnames[args->nfnames-1] = strdup(argv[optind]);
optind++;
}
- if ( args->allow_overlaps && args->phased_concat ) args->allow_overlaps = 0;
+ if ( args->allow_overlaps && args->phased_concat ) error("The options -a and -l should not be combined. Please run with -l only.\n");
if ( args->compact_PS && !args->phased_concat ) error("The -c option is intended only with -l\n");
if ( args->file_list )
{
/* vcfconcat.c -- Concatenate or combine VCF/BCF files.
- Copyright (C) 2013-2015 Genome Research Ltd.
+ Copyright (C) 2013-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#include <htslib/kseq.h>
#include <htslib/bgzf.h>
#include <htslib/tbx.h> // for hts_get_bgzfp()
+#include <htslib/thread_pool.h>
+#include <sys/time.h>
#include "bcftools.h"
typedef struct _args_t
char **argv, *output_fname, *file_list, **fnames, *remove_dups, *regions_list;
int argc, nfnames, allow_overlaps, phased_concat, regions_is_file;
- int compact_PS, phase_set_changed, naive_concat;
+ int compact_PS, phase_set_changed, naive_concat, naive_concat_trust_headers;
+ int verbose;
+ htsThreadPool *tpool;
}
args_t;
line = bcf_init();
}
+ if ( args->verbose ) fprintf(bcftools_stderr,"Checking the headers and starting positions of %d files\n", args->nfnames);
kstring_t str = {0,0,0};
int i, prev_chrid = -1;
for (i=0; i<args->nfnames; i++)
}
}
bcf_hdr_destroy(hdr);
- hts_close(fp);
+ if ( hts_close(fp)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fnames[i]);
}
free(str.s);
if ( line ) bcf_destroy(line);
if (args->record_cmd_line) bcf_hdr_append_version(args->out_hdr, args->argc, args->argv, "bcftools_concat");
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
- if ( args->n_threads ) hts_set_threads(args->out_fh, args->n_threads);
-
- bcf_hdr_write(args->out_fh, args->out_hdr);
-
- if ( args->allow_overlaps )
+ if ( args->allow_overlaps || args->phased_concat )
{
args->files = bcf_sr_init();
args->files->require_index = 1;
+ }
+ if ( args->n_threads )
+ {
+ if ( args->files )
+ {
+ if ( bcf_sr_set_threads(args->files, args->n_threads)<0 ) error("Failed to create threads\n");
+ args->tpool = args->files->p;
+ }
+ else
+ {
+ args->tpool = (htsThreadPool*) calloc(1, sizeof(htsThreadPool));
+ if ( !args->tpool ) error("Failed to allocate memory\n");
+ if ( !(args->tpool->pool = hts_tpool_init(args->n_threads)) ) error("Failed to initialize %d threads\n",args->n_threads);
+ }
+ hts_set_opt(args->out_fh, HTS_OPT_THREAD_POOL, args->tpool);
+ }
+ if ( bcf_hdr_write(args->out_fh, args->out_hdr)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname);
+
+ if ( args->allow_overlaps )
+ {
if ( args->regions_list )
{
if ( bcf_sr_set_regions(args->files, args->regions_list, args->regions_is_file)<0 )
args->nmism = (int*) calloc(bcf_hdr_nsamples(args->out_hdr),sizeof(int));
args->phase_qual = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
args->phase_set = (int32_t*) malloc(bcf_hdr_nsamples(args->out_hdr)*sizeof(int32_t));
- args->files = bcf_sr_init();
- args->files->require_index = 1;
args->ifname = 0;
}
}
static void destroy_data(args_t *args)
{
int i;
- for (i=0; i<args->nfnames; i++) free(args->fnames[i]);
- free(args->fnames);
- if ( args->files ) bcf_sr_destroy(args->files);
if ( args->out_fh )
{
if ( hts_close(args->out_fh)!=0 ) error("hts_close error\n");
}
+ if ( args->tpool && !args->files )
+ {
+ hts_tpool_destroy(args->tpool->pool);
+ free(args->tpool);
+ }
+ if ( args->files ) bcf_sr_destroy(args->files);
if ( args->out_hdr ) bcf_hdr_destroy(args->out_hdr);
free(args->seen_seq);
free(args->start_pos);
free(args->nmism);
free(args->phase_qual);
free(args->phase_set);
+ for (i=0; i<args->nfnames; i++) free(args->fnames[i]);
+ free(args->fnames);
}
int vcf_write_line(htsFile *fp, kstring_t *line);
{
if ( !gt_absent_warned )
{
- fprintf(bcftools_stderr,"GT is not present at %s:%d. (This warning is printed only once.)\n", bcf_seqname(ahdr,arec), arec->pos+1);
+ fprintf(bcftools_stderr,"GT is not present at %s:%"PRId64". (This warning is printed only once.)\n", bcf_seqname(ahdr,arec), (int64_t) arec->pos+1);
gt_absent_warned = 1;
}
continue;
{
if ( !gt_absent_warned )
{
- fprintf(bcftools_stderr,"GT is not present at %s:%d. (This warning is printed only once.)\n", bcf_seqname(bhdr,brec), brec->pos+1);
+ fprintf(bcftools_stderr,"GT is not present at %s:%"PRId64". (This warning is printed only once.)\n", bcf_seqname(bhdr,brec), (int64_t) brec->pos+1);
gt_absent_warned = 1;
}
continue;
bcf_update_format_int32(args->out_hdr,arec,"PS",args->phase_set,nsmpl);
args->phase_set_changed = 0;
}
- bcf_write(args->out_fh, args->out_hdr, arec);
+ if ( bcf_write(args->out_fh, args->out_hdr, arec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
- if ( arec->pos < args->prev_pos_check ) error("FIXME, disorder: %s:%d vs %d [1]\n", bcf_seqname(args->files->readers[0].header,arec),arec->pos+1,args->prev_pos_check+1);
+ if ( arec->pos < args->prev_pos_check ) error("FIXME, disorder: %s:%"PRId64" vs %d [1]\n", bcf_seqname(args->files->readers[0].header,arec),(int64_t) arec->pos+1,args->prev_pos_check+1);
args->prev_pos_check = arec->pos;
}
args->nswap = 0;
bcf_update_format_int32(args->out_hdr,brec,"PS",args->phase_set,nsmpl);
args->phase_set_changed = 0;
}
- bcf_write(args->out_fh, args->out_hdr, brec);
+ if ( bcf_write(args->out_fh, args->out_hdr, brec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
- if ( brec->pos < args->prev_pos_check ) error("FIXME, disorder: %s:%d vs %d [2]\n", bcf_seqname(args->files->readers[1].header,brec),brec->pos+1,args->prev_pos_check+1);
+ if ( brec->pos < args->prev_pos_check ) error("FIXME, disorder: %s:%"PRId64" vs %d [2]\n", bcf_seqname(args->files->readers[1].header,brec),(int64_t) brec->pos+1,args->prev_pos_check+1);
args->prev_pos_check = brec->pos;
}
args->nbuf = 0;
static void phased_push(args_t *args, bcf1_t *arec, bcf1_t *brec)
{
if ( arec && arec->errcode )
- error("Parse error at %s:%d, cannot proceed: %s\n", bcf_seqname(args->files->readers[0].header,arec),arec->pos+1, args->files->readers[0].fname);
+ error("Parse error at %s:%"PRId64", cannot proceed: %s\n", bcf_seqname(args->files->readers[0].header,arec),(int64_t) arec->pos+1, args->files->readers[0].fname);
if ( brec && brec->errcode )
- error("Parse error at %s:%d, cannot proceed: %s\n", bcf_seqname(args->files->readers[1].header,brec),brec->pos+1, args->files->readers[1].fname);
+ error("Parse error at %s:%"PRId64", cannot proceed: %s\n", bcf_seqname(args->files->readers[1].header,brec),(int64_t) brec->pos+1, args->files->readers[1].fname);
int i, nsmpl = bcf_hdr_nsamples(args->out_hdr);
int chr_id = bcf_hdr_name2id(args->out_hdr, bcf_seqname(args->files->readers[0].header,arec));
bcf_update_format_int32(args->out_hdr,arec,"PS",args->phase_set,nsmpl);
args->phase_set_changed = 0;
}
- bcf_write(args->out_fh, args->out_hdr, arec);
+ if ( bcf_write(args->out_fh, args->out_hdr, arec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
if ( arec->pos < args->prev_pos_check )
- error("FIXME, disorder: %s:%d in %s vs %d written [3]\n", bcf_seqname(args->files->readers[0].header,arec), arec->pos+1,args->files->readers[0].fname, args->prev_pos_check+1);
+ error("FIXME, disorder: %s:%"PRId64" in %s vs %d written [3]\n", bcf_seqname(args->files->readers[0].header,arec), (int64_t) arec->pos+1,args->files->readers[0].fname, args->prev_pos_check+1);
args->prev_pos_check = arec->pos;
return;
}
static void concat(args_t *args)
{
+ static int site_drop_warned = 0;
int i;
if ( args->phased_concat ) // phased concat
{
if ( !bcf_sr_has_line(args->files,0) ) // no input from the first reader
{
// We are assuming that there is a perfect overlap, sites which are not present in both files are dropped
- if ( ! bcf_sr_region_done(args->files,0) ) continue;
-
+ if ( ! bcf_sr_region_done(args->files,0) )
+ {
+ if ( !site_drop_warned )
+ {
+ fprintf(bcftools_stderr,
+ "Warning: Dropping the site %s:%"PRId64". The --ligate option is intended for VCFs with perfect\n"
+ " overlap, sites in overlapping regions present in one but missing in other are dropped.\n"
+ " This warning is printed only once.\n",
+ bcf_seqname(bcf_sr_get_header(args->files,1),bcf_sr_get_line(args->files,1)), (int64_t) bcf_sr_get_line(args->files,1)->pos+1
+ );
+ site_drop_warned = 1;
+ }
+ continue;
+ }
phased_flush(args);
bcf_sr_remove_reader(args->files, 0);
}
bcf1_t *line = bcf_sr_get_line(args->files,i);
if ( !line ) continue;
bcf_translate(args->out_hdr, args->files->readers[i].header, line);
- bcf_write1(args->out_fh, args->out_hdr, line);
+ if ( bcf_write1(args->out_fh, args->out_hdr, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
if ( args->remove_dups ) break;
}
}
}
else // concatenating
{
+ struct timeval t0, t1;
kstring_t tmp = {0,0,0};
int prev_chr_id = -1, prev_pos;
bcf1_t *line = bcf_init();
for (i=0; i<args->nfnames; i++)
{
- htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
- bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
+ if ( args->verbose )
+ {
+ fprintf(bcftools_stderr,"Concatenating %s", args->fnames[i]);
+ gettimeofday(&t0, NULL);
+ }
+ htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("\nFailed to open: %s\n", args->fnames[i]);
+ if ( args->n_threads ) hts_set_opt(fp, HTS_OPT_THREAD_POOL, args->tpool);
+ bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("\nFailed to parse header: %s\n", args->fnames[i]);
if ( !fp->is_bin && args->output_type&FT_VCF )
{
line->max_unpack = BCF_UN_STR;
tmp.l = 0;
kputsn(fp->line.s,str-fp->line.s,&tmp);
int chr_id = bcf_hdr_name2id(args->out_hdr, tmp.s);
- if ( chr_id<0 ) error("The sequence \"%s\" not defined in the header: %s\n(Quick workaround: index the file.)\n", tmp.s, args->fnames[i]);
+ if ( chr_id<0 ) error("\nThe sequence \"%s\" not defined in the header: %s\n(Quick workaround: index the file.)\n", tmp.s, args->fnames[i]);
if ( prev_chr_id!=chr_id )
{
prev_pos = -1;
int pos = strtol(str+1,&end,10) - 1;
if ( end==str+1 ) error("Could not parse line: %s\n", fp->line.s);
if ( prev_pos > pos )
- error("The chromosome block %s is not sorted, consider running with -a.\n", tmp.s);
+ error("\nThe chromosome block %s is not sorted, consider running with -a.\n", tmp.s);
args->seen_seq[chr_id] = 1;
prev_chr_id = chr_id;
- if ( vcf_write_line(args->out_fh, &fp->line)!=0 ) error("Failed to write %"PRIu64" bytes\n", (uint64_t)fp->line.l);
+ if ( vcf_write_line(args->out_fh, &fp->line)!=0 ) error("\nFailed to write %"PRIu64" bytes\n", (uint64_t)fp->line.l);
}
}
else
error("\nThe chromosome block %s is not contiguous, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
}
if ( prev_pos > line->pos )
- error("The chromosome block %s is not sorted, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
+ error("\nThe chromosome block %s is not sorted, consider running with -a.\n", bcf_seqname(args->out_hdr, line));
args->seen_seq[line->rid] = 1;
prev_chr_id = line->rid;
- if ( bcf_write(args->out_fh, args->out_hdr, line)!=0 ) error("Failed to write\n");
+ if ( bcf_write(args->out_fh, args->out_hdr, line)!=0 ) error("\nFailed to write\n");
}
}
bcf_hdr_destroy(hdr);
hts_close(fp);
+ if ( args->verbose )
+ {
+ gettimeofday(&t1, NULL);
+ double delta = (t1.tv_sec - t0.tv_sec) * 1e6 + (t1.tv_usec - t0.tv_usec);
+ fprintf(bcftools_stderr,"\t%f seconds\n",delta/1e6);
+ }
}
bcf_destroy(line);
free(tmp.s);
&& header[12] == 'B' && header[13] == 'C'
&& unpackInt16((uint8_t*)&header[14]) == 2) ? 0 : -1;
}
+static void _check_hrecs(const bcf_hdr_t *hdr0, const bcf_hdr_t *hdr, char *fname0, char *fname)
+{
+ int j;
+ for (j=0; j<hdr0->nhrec; j++)
+ {
+ bcf_hrec_t *hrec0 = hdr0->hrec[j];
+ if ( hrec0->type!=BCF_HL_FLT && hrec0->type!=BCF_HL_INFO && hrec0->type!=BCF_HL_FMT && hrec0->type!=BCF_HL_CTG ) continue; // skip fiels w/o IDX
+ int itag = bcf_hrec_find_key(hrec0, "ID");
+ bcf_hrec_t *hrec = bcf_hdr_get_hrec(hdr, hrec0->type, "ID", hrec0->vals[itag], NULL);
+
+ char *type = NULL;
+ if ( hrec0->type==BCF_HL_FLT ) type = "FILTER";
+ if ( hrec0->type==BCF_HL_INFO ) type = "INFO";
+ if ( hrec0->type==BCF_HL_FMT ) type = "FORMAT";
+ if ( hrec0->type==BCF_HL_CTG ) type = "contig";
+
+ if ( !hrec )
+ error("Cannot use --naive, incompatible headers, the tag %s/%s not present in %s\n",type,hrec0->vals[itag],fname);
+
+ int idx0 = bcf_hrec_find_key(hrec0, "IDX");
+ int idx = bcf_hrec_find_key(hrec, "IDX");
+ if ( idx0<0 || idx<0 )
+ error("fixme: unexpected IDX<0 for %s/%s in %s or %s\n",type,hrec0->vals[itag],fname0,fname);
+ if ( strcmp(hrec0->vals[idx0],hrec->vals[idx]) )
+ error("Cannot use --naive, use --naive-force instead: different order the tag %s/%s in %s vs %s\n",type,hrec0->vals[itag],fname0,fname);
+ }
+}
+static void naive_concat_check_headers(args_t *args)
+{
+ fprintf(bcftools_stderr,"Checking the headers of %d files.\n",args->nfnames);
+ bcf_hdr_t *hdr0 = NULL;
+ int i,j;
+ for (i=0; i<args->nfnames; i++)
+ {
+ htsFile *fp = hts_open(args->fnames[i], "r"); if ( !fp ) error("Failed to open: %s\n", args->fnames[i]);
+ bcf_hdr_t *hdr = bcf_hdr_read(fp); if ( !hdr ) error("Failed to parse header: %s\n", args->fnames[i]);
+ htsFormat type = *hts_get_format(fp);
+ hts_close(fp);
+
+ if ( i==0 )
+ {
+ hdr0 = hdr;
+ continue;
+ }
+
+ // check the samples
+ if ( bcf_hdr_nsamples(hdr0)!=bcf_hdr_nsamples(hdr) )
+ error("Cannot concatenate, different number of samples: %d vs %d in %s vs %s\n",bcf_hdr_nsamples(hdr0),bcf_hdr_nsamples(hdr),args->fnames[0],args->fnames[i]);
+ for (j=0; j<bcf_hdr_nsamples(hdr0); j++)
+ if ( strcmp(hdr0->samples[j],hdr->samples[j]) )
+ error("Cannot concatenate, different samples in %s vs %s\n",args->fnames[0],args->fnames[i]);
+
+ // if BCF, check if tag IDs are consistent in the dictionary of strings
+ if ( type.compression!=bgzf )
+ error("The --naive option works only for compressed BCFs or VCFs, sorry :-/\n");
+ if ( type.format==vcf )
+ {
+ bcf_hdr_destroy(hdr);
+ continue;
+ }
+
+ _check_hrecs(hdr0,hdr,args->fnames[0],args->fnames[i]);
+ _check_hrecs(hdr,hdr0,args->fnames[i],args->fnames[0]);
+
+ bcf_hdr_destroy(hdr);
+ }
+ if ( hdr0 ) bcf_hdr_destroy(hdr0);
+ fprintf(bcftools_stderr,"Done, the headers are compatible.\n");
+}
static void naive_concat(args_t *args)
{
+ if ( !args->naive_concat_trust_headers )
+ naive_concat_check_headers(args);
+
// only compressed BCF atm
BGZF *bgzf_out = bgzf_open(args->output_fname,"w");;
+ struct timeval t0, t1;
const size_t page_size = BGZF_MAX_BLOCK_SIZE;
uint8_t *buf = (uint8_t*) malloc(page_size);
kstring_t tmp = {0,0,0};
int i, file_types = 0;
for (i=0; i<args->nfnames; i++)
{
+ if ( args->verbose )
+ {
+ fprintf(bcftools_stderr,"Concatenating %s", args->fnames[i]);
+ gettimeofday(&t0, NULL);
+ }
htsFile *hts_fp = hts_open(args->fnames[i],"r");
- if ( !hts_fp ) error("Failed to open: %s\n", args->fnames[i]);
+ if ( !hts_fp ) error("\nFailed to open: %s\n", args->fnames[i]);
htsFormat type = *hts_get_format(hts_fp);
if ( type.compression!=bgzf )
- error("The --naive option works only for compressed BCFs or VCFs, sorry :-/\n");
+ error("\nThe --naive option works only for compressed BCFs or VCFs, sorry :-/\n");
file_types |= type.format==vcf ? 1 : 2;
if ( file_types==3 )
- error("The --naive option works only for compressed files of the same type, all BCFs or all VCFs :-/\n");
+ error("\nThe --naive option works only for compressed files of the same type, all BCFs or all VCFs :-/\n");
BGZF *fp = hts_get_bgzfp(hts_fp);
if ( !fp || bgzf_read_block(fp) != 0 || !fp->block_length )
- error("Failed to read %s: %s\n", args->fnames[i], strerror(errno));
+ error("\nFailed to read %s: %s\n", args->fnames[i], strerror(errno));
int nskip;
if ( type.format==bcf )
{
uint8_t magic[5];
- if ( bgzf_read(fp, magic, 5) != 5 ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
- if (strncmp((char*)magic, "BCF\2\2", 5) != 0) error("Invalid BCF magic string in %s\n", args->fnames[i]);
+ if ( bgzf_read(fp, magic, 5) != 5 ) error("\nFailed to read the BCF header in %s\n", args->fnames[i]);
+ if (strncmp((char*)magic, "BCF\2\2", 5) != 0) error("\nInvalid BCF magic string in %s\n", args->fnames[i]);
- if ( bgzf_read(fp, &tmp.l, 4) != 4 ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
+ if ( bgzf_read(fp, &tmp.l, 4) != 4 ) error("\nFailed to read the BCF header in %s\n", args->fnames[i]);
hts_expand(char,tmp.l,tmp.m,tmp.s);
- if ( bgzf_read(fp, tmp.s, tmp.l) != tmp.l ) error("Failed to read the BCF header in %s\n", args->fnames[i]);
+ if ( bgzf_read(fp, tmp.s, tmp.l) != tmp.l ) error("\nFailed to read the BCF header in %s\n", args->fnames[i]);
// write only the first header
if ( i==0 )
{
- if ( bgzf_write(bgzf_out, "BCF\2\2", 5) !=5 ) error("Failed to write %d bytes to %s\n", 5,args->output_fname);
- if ( bgzf_write(bgzf_out, &tmp.l, 4) !=4 ) error("Failed to write %d bytes to %s\n", 4,args->output_fname);
- if ( bgzf_write(bgzf_out, tmp.s, tmp.l) != tmp.l) error("Failed to write %"PRId64" bytes to %s\n", (uint64_t)tmp.l,args->output_fname);
+ if ( bgzf_write(bgzf_out, "BCF\2\2", 5) !=5 ) error("\nFailed to write %d bytes to %s\n", 5,args->output_fname);
+ if ( bgzf_write(bgzf_out, &tmp.l, 4) !=4 ) error("\nFailed to write %d bytes to %s\n", 4,args->output_fname);
+ if ( bgzf_write(bgzf_out, tmp.s, tmp.l) != tmp.l) error("\nFailed to write %"PRId64" bytes to %s\n", (uint64_t)tmp.l,args->output_fname);
}
nskip = fp->block_offset;
}
else
{
nskip = print_vcf_gz_header(fp, bgzf_out, i==0?1:0, &tmp);
- if ( nskip==-1 ) error("Error reading %s\n", args->fnames[i]);
+ if ( nskip==-1 ) error("\nError reading %s\n", args->fnames[i]);
}
// Output all non-header data that were read together with the header block
if ( fp->block_length - nskip > 0 )
{
- if ( bgzf_write(bgzf_out, (char *)fp->uncompressed_block+nskip, fp->block_length-nskip)<0 ) error("Error: %d\n",fp->errcode);
+ if ( bgzf_write(bgzf_out, (char *)fp->uncompressed_block+nskip, fp->block_length-nskip)<0 ) error("\nError: %d\n",fp->errcode);
}
- if ( bgzf_flush(bgzf_out)<0 ) error("Error: %d\n",bgzf_out->errcode);
+ if ( bgzf_flush(bgzf_out)<0 ) error("\nError: %d\n",bgzf_out->errcode);
// Stream the rest of the file as it is, without recompressing, but remove BGZF EOF blocks
{
nread = bgzf_raw_read(fp, buf, nheader);
if ( !nread ) break;
- if ( nread != nheader || check_header(buf)!=0 ) error("Could not parse the header of a bgzf block: %s\n",args->fnames[i]);
+ if ( nread != nheader || check_header(buf)!=0 ) error("\nCould not parse the header of a bgzf block: %s\n",args->fnames[i]);
nblock = unpackInt16(buf+16) + 1;
assert( nblock <= page_size && nblock >= nheader );
nread += bgzf_raw_read(fp, buf+nheader, nblock - nheader);
- if ( nread!=nblock ) error("Could not read %"PRId64" bytes: %s\n",(uint64_t)nblock,args->fnames[i]);
+ if ( nread!=nblock ) error("\nCould not read %"PRId64" bytes: %s\n",(uint64_t)nblock,args->fnames[i]);
if ( nread==neof && !memcmp(buf,eof,neof) ) continue;
nwr = bgzf_raw_write(bgzf_out, buf, nread);
- if ( nwr != nread ) error("Write failed, wrote %"PRId64" instead of %d bytes.\n", (uint64_t)nwr,(int)nread);
+ if ( nwr != nread ) error("\nWrite failed, wrote %"PRId64" instead of %d bytes.\n", (uint64_t)nwr,(int)nread);
+ }
+ if (hts_close(hts_fp)) error("\nClose failed: %s\n",args->fnames[i]);
+ if ( args->verbose )
+ {
+ gettimeofday(&t1, NULL);
+ double delta = (t1.tv_sec - t0.tv_sec) * 1e6 + (t1.tv_usec - t0.tv_usec);
+ fprintf(bcftools_stderr,"\t%f seconds\n",delta/1e6);
}
- if (hts_close(hts_fp)) error("Close failed: %s\n",args->fnames[i]);
}
free(buf);
free(tmp.s);
fprintf(bcftools_stderr, " VCF into one. The input files must be sorted by chr and position. The files\n");
fprintf(bcftools_stderr, " must be given in the correct order to produce sorted VCF on output unless\n");
fprintf(bcftools_stderr, " the -a, --allow-overlaps option is specified. With the --naive option, the files\n");
- fprintf(bcftools_stderr, " are concatenated without being recompressed, which is very fast but dangerous\n");
- fprintf(bcftools_stderr, " if the BCF headers differ.\n");
+ fprintf(bcftools_stderr, " are concatenated without being recompressed, which is very fast.\n");
fprintf(bcftools_stderr, "Usage: bcftools concat [options] <A.vcf.gz> [<B.vcf.gz> [...]]\n");
fprintf(bcftools_stderr, "\n");
fprintf(bcftools_stderr, "Options:\n");
fprintf(bcftools_stderr, " -f, --file-list <file> Read the list of files from a file.\n");
fprintf(bcftools_stderr, " -l, --ligate Ligate phased VCFs by matching phase at overlapping haplotypes\n");
fprintf(bcftools_stderr, " --no-version Do not append version and command line to the header\n");
- fprintf(bcftools_stderr, " -n, --naive Concatenate files without recompression (dangerous, use with caution)\n");
+ fprintf(bcftools_stderr, " -n, --naive Concatenate files without recompression, a header check compatibility is performed\n");
+ fprintf(bcftools_stderr, " --naive-force Same as --naive, but header compatibility is not checked. Dangerous, use with caution.\n");
fprintf(bcftools_stderr, " -o, --output <file> Write output to a file [standard output]\n");
fprintf(bcftools_stderr, " -O, --output-type <b|u|z|v> b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n");
fprintf(bcftools_stderr, " -q, --min-PQ <int> Break phase set if phasing quality is lower than <int> [30]\n");
fprintf(bcftools_stderr, " -r, --regions <region> Restrict to comma-separated list of regions\n");
fprintf(bcftools_stderr, " -R, --regions-file <file> Restrict to regions listed in a file\n");
- fprintf(bcftools_stderr, " --threads <int> Number of extra output compression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> Use multithreading with <int> worker threads [0]\n");
+ fprintf(bcftools_stderr, " -v, --verbose <0|1> Set verbosity level [1]\n");
fprintf(bcftools_stderr, "\n");
exit(1);
}
args->n_threads = 0;
args->record_cmd_line = 1;
args->min_PQ = 30;
+ args->verbose = 1;
static struct option loptions[] =
{
+ {"verbose",required_argument,NULL,'v'},
{"naive",no_argument,NULL,'n'},
+ {"naive-force",no_argument,NULL,7},
{"compact-PS",no_argument,NULL,'c'},
{"regions",required_argument,NULL,'r'},
{"regions-file",required_argument,NULL,'R'},
{NULL,0,NULL,0}
};
char *tmp;
- while ((c = getopt_long(argc, argv, "h:?o:O:f:alq:Dd:r:R:cn",loptions,NULL)) >= 0)
+ while ((c = getopt_long(argc, argv, "h:?o:O:f:alq:Dd:r:R:cnv:",loptions,NULL)) >= 0)
{
switch (c) {
case 'c': args->compact_PS = 1; break;
break;
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
+ case 7 : args->naive_concat = 1; args->naive_concat_trust_headers = 1; break;
+ case 'v':
+ args->verbose = strtol(optarg, 0, 0);
+ error("Error: currently only --verbose 0 or --verbose 1 is supported\n");
+ break;
case 'h':
case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
args->fnames[args->nfnames-1] = strdup(argv[optind]);
optind++;
}
- if ( args->allow_overlaps && args->phased_concat ) args->allow_overlaps = 0;
+ if ( args->allow_overlaps && args->phased_concat ) error("The options -a and -l should not be combined. Please run with -l only.\n");
if ( args->compact_PS && !args->phased_concat ) error("The -c option is intended only with -l\n");
if ( args->file_list )
{
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
+#include <inttypes.h>
#include <htslib/faidx.h>
#include <htslib/vcf.h>
#include <htslib/bgzf.h>
htsFile *out_fh = hts_open(args->outfname,hts_bcf_wmode(args->output_type));
if ( out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->outfname, strerror(errno));
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
- bcf_hdr_write(out_fh,args->header);
+ if ( bcf_hdr_write(out_fh,args->header)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->outfname);
bcf1_t *rec = bcf_init();
nsamples -= 2;
bcf_clear(rec);
args->n.total++;
if ( !tsv_parse(tsv, rec, line.s) )
- bcf_write(out_fh, args->header, rec);
+ {
+ if ( bcf_write(out_fh, args->header, rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
+ }
else
error("Error occurred while parsing: %s\n", line.s);
}
htsFile *out_fh = hts_open(args->outfname,hts_bcf_wmode(args->output_type));
if ( out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->outfname, strerror(errno));
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
- bcf_hdr_write(out_fh,args->header);
+ if ( bcf_hdr_write(out_fh,args->header)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->outfname);
bcf1_t *rec = bcf_init();
args->gts = (int32_t *) malloc(sizeof(int32_t)*nsamples*2);
if ( tsv_parse(hap_tsv, rec, line.s) )
error("Error occurred while parsing %s: %s\n", hap_fname,line.s);
- bcf_write(out_fh, args->header, rec);
+ if ( bcf_write(out_fh, args->header, rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
if ( hts_getline(leg_fh, KS_SEP_LINE, &line)<=0 )
{
htsFile *out_fh = hts_open(args->outfname,hts_bcf_wmode(args->output_type));
if ( out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->outfname, strerror(errno));
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
- bcf_hdr_write(out_fh,args->header);
+ if ( bcf_hdr_write(out_fh,args->header)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
bcf1_t *rec = bcf_init();
nsamples -= 2;
bcf_clear(rec);
args->n.total++;
if ( !tsv_parse(tsv, rec, line.s) )
- bcf_write(out_fh, args->header, rec);
+ {
+ if ( bcf_write(out_fh, args->header, rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
+ }
else
error("Error occurred while parsing: %s\n", line.s);
}
if (legend_fname) {
str.l = 0;
if ( args->output_vcf_ids && (line->d.id[0]!='.' || line->d.id[1]!=0) )
- ksprintf(&str, "%s %d %s %s\n", line->d.id, line->pos+1, line->d.allele[0], line->d.allele[1]);
+ ksprintf(&str, "%s %"PRId64" %s %s\n", line->d.id, (int64_t) line->pos+1, line->d.allele[0], line->d.allele[1]);
else
- ksprintf(&str, "%s:%d_%s_%s %d %s %s\n", bcf_seqname(args->header, line), line->pos+1, line->d.allele[0], line->d.allele[1], line->pos+1, line->d.allele[0], line->d.allele[1]);
+ ksprintf(&str, "%s:%"PRId64"_%s_%s %"PRId64" %s %s\n", bcf_seqname(args->header, line), (int64_t) line->pos+1, line->d.allele[0], line->d.allele[1], (int64_t) line->pos+1, line->d.allele[0], line->d.allele[1]);
// write legend file
ret = bgzf_write(lout, str.s, str.l);
int len;
char *ref = faidx_fetch_seq(args->ref, (char*)bcf_hdr_id2name(args->header,rec->rid), rec->pos, rec->pos, &len);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", bcf_hdr_id2name(args->header,rec->rid), rec->pos+1);
+ if ( !ref ) error("faidx_fetch_seq failed at %s:%"PRId64"\n", bcf_hdr_id2name(args->header,rec->rid),(int64_t) rec->pos+1);
int nals = 1, alleles[5] = { -1, -1, -1, -1, -1 }; // a,c,g,t,n
ref[0] = toupper(ref[0]);
if ( i>0 )
{
ret = tsv_next(tsv);
- if ( ret==-1 ) error("Too few columns for %d samples at %s:%d\n", rec->n_sample,bcf_hdr_id2name(args->header,rec->rid), rec->pos+1);
+ if ( ret==-1 ) error("Too few columns for %d samples at %s:%"PRId64"\n", rec->n_sample,bcf_hdr_id2name(args->header,rec->rid),(int64_t) rec->pos+1);
}
ret = tsv_setter_aa1(args, tsv->ss, tsv->se, alleles, &nals, iref, args->gts+i*2);
- if ( ret==-1 ) error("Error parsing the site %s:%d, expected two characters\n", bcf_hdr_id2name(args->header,rec->rid), rec->pos+1);
+ if ( ret==-1 ) error("Error parsing the site %s:%"PRId64", expected two characters\n", bcf_hdr_id2name(args->header,rec->rid),(int64_t) rec->pos+1);
if ( ret==-2 )
{
// something else than a SNP
htsFile *out_fh = hts_open(args->outfname,hts_bcf_wmode(args->output_type));
if ( out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->outfname, strerror(errno));
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
- bcf_hdr_write(out_fh,args->header);
+ if ( bcf_hdr_write(out_fh,args->header)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
tsv_t *tsv = tsv_init(args->columns ? args->columns : "ID,CHROM,POS,AA");
if ( tsv_register(tsv, "CHROM", tsv_setter_chrom, args->header) < 0 ) error("Expected CHROM column\n");
args->n.total++;
if ( !tsv_parse(tsv, rec, line.s) )
- bcf_write(out_fh, args->header, rec);
+ {
+ if ( bcf_write(out_fh, args->header, rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
+ }
else
args->n.skipped++;
}
free(line.s);
bcf_hdr_destroy(args->header);
- hts_close(out_fh);
+ if ( hts_close(out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->outfname);
tsv_destroy(tsv);
bcf_destroy(rec);
free(args->str.s);
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
bcf_hdr_t *hdr = bcf_sr_get_header(args->files,0);
- bcf_hdr_write(out_fh,hdr);
+ if ( bcf_hdr_write(out_fh,hdr)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
while ( bcf_sr_next_line(args->files) )
{
if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
if ( !pass ) continue;
}
- bcf_write(out_fh,hdr,line);
+ if ( bcf_write(out_fh,hdr,line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
}
- hts_close(out_fh);
+ if ( hts_close(out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->outfname);
}
static void gvcf_to_vcf(args_t *args)
bcf_hdr_t *hdr = bcf_sr_get_header(args->files,0);
if (args->record_cmd_line) bcf_hdr_append_version(hdr, args->argc, args->argv, "bcftools_convert");
- bcf_hdr_write(out_fh,hdr);
+ if ( bcf_hdr_write(out_fh,hdr)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
int32_t *itmp = NULL, nitmp = 0;
if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
if ( !pass )
{
- bcf_write(out_fh,hdr,line);
+ if ( bcf_write(out_fh,hdr,line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
continue;
}
}
// no gVCF compatible alleles
if (gallele<0)
{
- bcf_write(out_fh,hdr,line);
+ if ( bcf_write(out_fh,hdr,line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
continue;
}
if ( nend!=1 )
{
// No INFO/END => not gVCF record
- bcf_write(out_fh,hdr,line);
+ if ( bcf_write(out_fh,hdr,line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
continue;
}
bcf_update_info_int32(hdr,line,"END",NULL,0);
{
line->pos = pos;
char *ref = faidx_fetch_seq(args->ref, (char*)bcf_hdr_id2name(hdr,line->rid), line->pos, line->pos, &len);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", bcf_hdr_id2name(hdr,line->rid), line->pos+1);
+ if ( !ref ) error("faidx_fetch_seq failed at %s:%"PRId64"\n", bcf_hdr_id2name(hdr,line->rid),(int64_t) line->pos+1);
strncpy(line->d.allele[0],ref,len);
- bcf_write(out_fh,hdr,line);
+ if ( bcf_write(out_fh,hdr,line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
free(ref);
}
}
free(itmp);
- hts_close(out_fh);
+ if ( hts_close(out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->outfname);
}
static void usage(void)
fprintf(stderr, " --no-version do not append version and command line to the header\n");
fprintf(stderr, " -o, --output <file> output file name [stdout]\n");
fprintf(stderr, " -O, --output-type <b|u|z|v> b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n");
- fprintf(stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(stderr, "\n");
fprintf(stderr, "GEN/SAMPLE conversion (input/output from IMPUTE2):\n");
fprintf(stderr, " -G, --gensample2vcf <...> <prefix>|<gen-file>,<sample-file>\n");
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 10 : args->record_cmd_line = 0; break;
case 11 : args->sex_fname = optarg; break;
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
+#include <inttypes.h>
#include <htslib/faidx.h>
#include <htslib/vcf.h>
#include <htslib/bgzf.h>
htsFile *out_fh = hts_open(args->outfname,hts_bcf_wmode(args->output_type));
if ( out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->outfname, strerror(errno));
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
- bcf_hdr_write(out_fh,args->header);
+ if ( bcf_hdr_write(out_fh,args->header)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->outfname);
bcf1_t *rec = bcf_init();
nsamples -= 2;
bcf_clear(rec);
args->n.total++;
if ( !tsv_parse(tsv, rec, line.s) )
- bcf_write(out_fh, args->header, rec);
+ {
+ if ( bcf_write(out_fh, args->header, rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
+ }
else
error("Error occurred while parsing: %s\n", line.s);
}
htsFile *out_fh = hts_open(args->outfname,hts_bcf_wmode(args->output_type));
if ( out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->outfname, strerror(errno));
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
- bcf_hdr_write(out_fh,args->header);
+ if ( bcf_hdr_write(out_fh,args->header)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->outfname);
bcf1_t *rec = bcf_init();
args->gts = (int32_t *) malloc(sizeof(int32_t)*nsamples*2);
if ( tsv_parse(hap_tsv, rec, line.s) )
error("Error occurred while parsing %s: %s\n", hap_fname,line.s);
- bcf_write(out_fh, args->header, rec);
+ if ( bcf_write(out_fh, args->header, rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
if ( hts_getline(leg_fh, KS_SEP_LINE, &line)<=0 )
{
htsFile *out_fh = hts_open(args->outfname,hts_bcf_wmode(args->output_type));
if ( out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->outfname, strerror(errno));
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
- bcf_hdr_write(out_fh,args->header);
+ if ( bcf_hdr_write(out_fh,args->header)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
bcf1_t *rec = bcf_init();
nsamples -= 2;
bcf_clear(rec);
args->n.total++;
if ( !tsv_parse(tsv, rec, line.s) )
- bcf_write(out_fh, args->header, rec);
+ {
+ if ( bcf_write(out_fh, args->header, rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
+ }
else
error("Error occurred while parsing: %s\n", line.s);
}
if (legend_fname) {
str.l = 0;
if ( args->output_vcf_ids && (line->d.id[0]!='.' || line->d.id[1]!=0) )
- ksprintf(&str, "%s %d %s %s\n", line->d.id, line->pos+1, line->d.allele[0], line->d.allele[1]);
+ ksprintf(&str, "%s %"PRId64" %s %s\n", line->d.id, (int64_t) line->pos+1, line->d.allele[0], line->d.allele[1]);
else
- ksprintf(&str, "%s:%d_%s_%s %d %s %s\n", bcf_seqname(args->header, line), line->pos+1, line->d.allele[0], line->d.allele[1], line->pos+1, line->d.allele[0], line->d.allele[1]);
+ ksprintf(&str, "%s:%"PRId64"_%s_%s %"PRId64" %s %s\n", bcf_seqname(args->header, line), (int64_t) line->pos+1, line->d.allele[0], line->d.allele[1], (int64_t) line->pos+1, line->d.allele[0], line->d.allele[1]);
// write legend file
ret = bgzf_write(lout, str.s, str.l);
int len;
char *ref = faidx_fetch_seq(args->ref, (char*)bcf_hdr_id2name(args->header,rec->rid), rec->pos, rec->pos, &len);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", bcf_hdr_id2name(args->header,rec->rid), rec->pos+1);
+ if ( !ref ) error("faidx_fetch_seq failed at %s:%"PRId64"\n", bcf_hdr_id2name(args->header,rec->rid),(int64_t) rec->pos+1);
int nals = 1, alleles[5] = { -1, -1, -1, -1, -1 }; // a,c,g,t,n
ref[0] = toupper(ref[0]);
if ( i>0 )
{
ret = tsv_next(tsv);
- if ( ret==-1 ) error("Too few columns for %d samples at %s:%d\n", rec->n_sample,bcf_hdr_id2name(args->header,rec->rid), rec->pos+1);
+ if ( ret==-1 ) error("Too few columns for %d samples at %s:%"PRId64"\n", rec->n_sample,bcf_hdr_id2name(args->header,rec->rid),(int64_t) rec->pos+1);
}
ret = tsv_setter_aa1(args, tsv->ss, tsv->se, alleles, &nals, iref, args->gts+i*2);
- if ( ret==-1 ) error("Error parsing the site %s:%d, expected two characters\n", bcf_hdr_id2name(args->header,rec->rid), rec->pos+1);
+ if ( ret==-1 ) error("Error parsing the site %s:%"PRId64", expected two characters\n", bcf_hdr_id2name(args->header,rec->rid),(int64_t) rec->pos+1);
if ( ret==-2 )
{
// something else than a SNP
htsFile *out_fh = hts_open(args->outfname,hts_bcf_wmode(args->output_type));
if ( out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->outfname, strerror(errno));
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
- bcf_hdr_write(out_fh,args->header);
+ if ( bcf_hdr_write(out_fh,args->header)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
tsv_t *tsv = tsv_init(args->columns ? args->columns : "ID,CHROM,POS,AA");
if ( tsv_register(tsv, "CHROM", tsv_setter_chrom, args->header) < 0 ) error("Expected CHROM column\n");
args->n.total++;
if ( !tsv_parse(tsv, rec, line.s) )
- bcf_write(out_fh, args->header, rec);
+ {
+ if ( bcf_write(out_fh, args->header, rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
+ }
else
args->n.skipped++;
}
free(line.s);
bcf_hdr_destroy(args->header);
- hts_close(out_fh);
+ if ( hts_close(out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->outfname);
tsv_destroy(tsv);
bcf_destroy(rec);
free(args->str.s);
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
bcf_hdr_t *hdr = bcf_sr_get_header(args->files,0);
- bcf_hdr_write(out_fh,hdr);
+ if ( bcf_hdr_write(out_fh,hdr)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
while ( bcf_sr_next_line(args->files) )
{
if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
if ( !pass ) continue;
}
- bcf_write(out_fh,hdr,line);
+ if ( bcf_write(out_fh,hdr,line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
}
- hts_close(out_fh);
+ if ( hts_close(out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->outfname);
}
static void gvcf_to_vcf(args_t *args)
bcf_hdr_t *hdr = bcf_sr_get_header(args->files,0);
if (args->record_cmd_line) bcf_hdr_append_version(hdr, args->argc, args->argv, "bcftools_convert");
- bcf_hdr_write(out_fh,hdr);
+ if ( bcf_hdr_write(out_fh,hdr)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
int32_t *itmp = NULL, nitmp = 0;
if ( args->filter_logic & FLT_EXCLUDE ) pass = pass ? 0 : 1;
if ( !pass )
{
- bcf_write(out_fh,hdr,line);
+ if ( bcf_write(out_fh,hdr,line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
continue;
}
}
// no gVCF compatible alleles
if (gallele<0)
{
- bcf_write(out_fh,hdr,line);
+ if ( bcf_write(out_fh,hdr,line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
continue;
}
if ( nend!=1 )
{
// No INFO/END => not gVCF record
- bcf_write(out_fh,hdr,line);
+ if ( bcf_write(out_fh,hdr,line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
continue;
}
bcf_update_info_int32(hdr,line,"END",NULL,0);
{
line->pos = pos;
char *ref = faidx_fetch_seq(args->ref, (char*)bcf_hdr_id2name(hdr,line->rid), line->pos, line->pos, &len);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", bcf_hdr_id2name(hdr,line->rid), line->pos+1);
+ if ( !ref ) error("faidx_fetch_seq failed at %s:%"PRId64"\n", bcf_hdr_id2name(hdr,line->rid),(int64_t) line->pos+1);
strncpy(line->d.allele[0],ref,len);
- bcf_write(out_fh,hdr,line);
+ if ( bcf_write(out_fh,hdr,line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->outfname);
free(ref);
}
}
free(itmp);
- hts_close(out_fh);
+ if ( hts_close(out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->outfname);
}
static void usage(void)
fprintf(bcftools_stderr, " --no-version do not append version and command line to the header\n");
fprintf(bcftools_stderr, " -o, --output <file> output file name [bcftools_stdout]\n");
fprintf(bcftools_stderr, " -O, --output-type <b|u|z|v> b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(bcftools_stderr, "\n");
fprintf(bcftools_stderr, "GEN/SAMPLE conversion (input/output from IMPUTE2):\n");
fprintf(bcftools_stderr, " -G, --gensample2vcf <...> <prefix>|<gen-file>,<sample-file>\n");
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 10 : args->record_cmd_line = 0; break;
case 11 : args->sex_fname = optarg; break;
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
if ( args->snp_gap && rec->d.flt[j]==args->SnpGap_id ) { pass = 0; break; }
}
}
- if ( pass ) bcf_write1(args->out_fh, args->hdr, rec);
+ if ( pass && bcf_write1(args->out_fh, args->hdr, rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
}
if ( k_flush || !line )
{
// Select the best indel from the cluster of k_flush indels
- int k = 0, max_ac = -1, imax_ac = -1;
+ int k = 0, max_ac = -1, imax_ac = -1, max_qual = -1, imax_qual = -1;
for (i=-1; rbuf_next(&args->rbuf,&i) && k<k_flush; )
{
k++;
hts_expand(int, rec->n_allele, args->ntmpi, args->tmpi);
int ret = bcf_calc_ac(args->hdr, rec, args->tmpi, BCF_UN_ALL);
if ( imax_ac==-1 || (ret && max_ac < args->tmpi[1]) ) { max_ac = args->tmpi[1]; imax_ac = i; }
+ if ( imax_qual==-1 || max_qual < rec->qual ) { max_qual = rec->qual; imax_qual = i; }
}
- // Filter all but the best indel (with max AF or first if AF not available)
+ // Filter all but the best indel (with the best QUAL, bigger AC, or take the first if neither QUAL nor AC are available)
k = 0;
for (i=-1; rbuf_next(&args->rbuf,&i) && k<k_flush; )
{
bcf1_t *rec = args->rbuf_lines[i];
if ( !(rec->d.var_type & IndelGap_set) ) continue;
rec->d.var_type |= IndelGap_flush;
- if ( i!=imax_ac ) bcf_add_filter(args->hdr, args->rbuf_lines[i], args->IndelGap_id);
+
+ int do_filter = 0;
+ if ( max_qual>0 )
+ {
+ if ( i!=imax_qual ) do_filter = 1;
+ }
+ else if ( i!=imax_ac ) do_filter = 1;
+ if ( do_filter ) bcf_add_filter(args->hdr, args->rbuf_lines[i], args->IndelGap_id);
}
}
}
fprintf(stderr, " -S, --set-GTs <.|0> set genotypes of failed samples to missing (.) or ref (0)\n");
fprintf(stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
- fprintf(stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(stderr, "\n");
exit(1);
}
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
case 'h':
- case '?': usage(args);
+ case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
}
if ( bcf_sr_set_targets(args->files, args->targets_list,targets_is_file, 0)<0 )
error("Failed to read the targets: %s\n", args->targets_list);
}
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
init_data(args);
- bcf_hdr_write(args->out_fh, args->hdr);
+ if ( bcf_hdr_write(args->out_fh, args->hdr)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname);
while ( bcf_sr_next_line(args->files) )
{
bcf1_t *line = bcf_sr_get_line(args->files, 0);
}
if ( args->set_gts ) set_genotypes(args, line, pass);
if ( !args->rbuf_lines )
- bcf_write1(args->out_fh, args->hdr, line);
+ {
+ if ( bcf_write1(args->out_fh, args->hdr, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
+ }
else
buffered_filters(args, line);
}
}
buffered_filters(args, NULL);
- hts_close(args->out_fh);
+ if ( hts_close(args->out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
destroy_data(args);
bcf_sr_destroy(args->files);
free(args);
if ( args->snp_gap && rec->d.flt[j]==args->SnpGap_id ) { pass = 0; break; }
}
}
- if ( pass ) bcf_write1(args->out_fh, args->hdr, rec);
+ if ( pass && bcf_write1(args->out_fh, args->hdr, rec)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
}
if ( k_flush || !line )
{
// Select the best indel from the cluster of k_flush indels
- int k = 0, max_ac = -1, imax_ac = -1;
+ int k = 0, max_ac = -1, imax_ac = -1, max_qual = -1, imax_qual = -1;
for (i=-1; rbuf_next(&args->rbuf,&i) && k<k_flush; )
{
k++;
hts_expand(int, rec->n_allele, args->ntmpi, args->tmpi);
int ret = bcf_calc_ac(args->hdr, rec, args->tmpi, BCF_UN_ALL);
if ( imax_ac==-1 || (ret && max_ac < args->tmpi[1]) ) { max_ac = args->tmpi[1]; imax_ac = i; }
+ if ( imax_qual==-1 || max_qual < rec->qual ) { max_qual = rec->qual; imax_qual = i; }
}
- // Filter all but the best indel (with max AF or first if AF not available)
+ // Filter all but the best indel (with the best QUAL, bigger AC, or take the first if neither QUAL nor AC are available)
k = 0;
for (i=-1; rbuf_next(&args->rbuf,&i) && k<k_flush; )
{
bcf1_t *rec = args->rbuf_lines[i];
if ( !(rec->d.var_type & IndelGap_set) ) continue;
rec->d.var_type |= IndelGap_flush;
- if ( i!=imax_ac ) bcf_add_filter(args->hdr, args->rbuf_lines[i], args->IndelGap_id);
+
+ int do_filter = 0;
+ if ( max_qual>0 )
+ {
+ if ( i!=imax_qual ) do_filter = 1;
+ }
+ else if ( i!=imax_ac ) do_filter = 1;
+ if ( do_filter ) bcf_add_filter(args->hdr, args->rbuf_lines[i], args->IndelGap_id);
}
}
}
fprintf(bcftools_stderr, " -S, --set-GTs <.|0> set genotypes of failed samples to missing (.) or ref (0)\n");
fprintf(bcftools_stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(bcftools_stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(bcftools_stderr, "\n");
exit(1);
}
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
case 'h':
- case '?': usage(args);
+ case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
}
if ( bcf_sr_set_targets(args->files, args->targets_list,targets_is_file, 0)<0 )
error("Failed to read the targets: %s\n", args->targets_list);
}
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
init_data(args);
- bcf_hdr_write(args->out_fh, args->hdr);
+ if ( bcf_hdr_write(args->out_fh, args->hdr)!=0 ) error("[%s] Error: cannot write the header to %s\n", __func__,args->output_fname);
while ( bcf_sr_next_line(args->files) )
{
bcf1_t *line = bcf_sr_get_line(args->files, 0);
}
if ( args->set_gts ) set_genotypes(args, line, pass);
if ( !args->rbuf_lines )
- bcf_write1(args->out_fh, args->hdr, line);
+ {
+ if ( bcf_write1(args->out_fh, args->hdr, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
+ }
else
buffered_filters(args, line);
}
}
buffered_filters(args, NULL);
- hts_close(args->out_fh);
+ if ( hts_close(args->out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
destroy_data(args);
bcf_sr_destroy(args->files);
free(args);
int fake_PL = args->no_PLs ? args->no_PLs : 99; // with 1, discordance is the number of non-matching GTs
int nsm_gt, i;
if ( (nsm_gt=bcf_get_genotypes(hdr, line, &args->tmp_arr, &args->ntmp_arr)) <= 0 )
- error("GT not present at %s:%d?\n", hdr->id[BCF_DT_CTG][line->rid].key, line->pos+1);
+ error("GT not present at %s:%"PRId64"?\n", hdr->id[BCF_DT_CTG][line->rid].key, (int64_t) line->pos+1);
nsm_gt /= bcf_hdr_nsamples(hdr);
int npl = line->n_allele*(line->n_allele+1)/2;
hts_expand(int,npl*bcf_hdr_nsamples(hdr),args->npl_arr,args->pl_arr);
// Target genotypes
int ngt, npl;
if ( (ngt=bcf_get_genotypes(args->gt_hdr, gt_line, >_arr, &ngt_arr)) <= 0 )
- error("GT not present at %s:%d?", args->gt_hdr->id[BCF_DT_CTG][gt_line->rid].key, gt_line->pos+1);
+ error("GT not present at %s:%"PRId64"?", args->gt_hdr->id[BCF_DT_CTG][gt_line->rid].key, (int64_t) gt_line->pos+1);
ngt /= bcf_hdr_nsamples(args->gt_hdr);
if ( ngt!=2 ) continue; // checking only diploid genotypes
npl = fake_PLs(args, args->sm_hdr, sm_line);
}
else
- error("PL not present at %s:%d?\n", args->sm_hdr->id[BCF_DT_CTG][sm_line->rid].key, sm_line->pos+1);
+ error("PL not present at %s:%"PRId64"?\n", args->sm_hdr->id[BCF_DT_CTG][sm_line->rid].key, (int64_t) sm_line->pos+1);
}
else
npl /= bcf_hdr_nsamples(args->sm_hdr);
int a = bcf_gt_allele(gt_ptr[0]);
int b = bcf_gt_allele(gt_ptr[1]);
if ( args->hom_only && a!=b ) continue; // heterozygous genotype
- fprintf(fp, "SC\t%s\t%d", args->gt_hdr->id[BCF_DT_CTG][gt_line->rid].key, gt_line->pos+1);
+ fprintf(fp, "SC\t%s\t%"PRId64, args->gt_hdr->id[BCF_DT_CTG][gt_line->rid].key, (int64_t) gt_line->pos+1);
for (i=0; i<gt_line->n_allele; i++) fprintf(fp, "%c%s", i==0?'\t':',', gt_line->d.allele[i]);
fprintf(fp, "\t%s/%s", a>=0 ? gt_line->d.allele[a] : ".", b>=0 ? gt_line->d.allele[b] : ".");
fprintf(fp, "\t%f", args->lks[query_isample]-prev_lk);
if ( args->plot )
{
- fclose(fp);
+ if ( fclose(fp)!=0 ) error("[%s] Error: close failed\n", __func__);
plot_check(args, args->target_sample ? args->target_sample : "", args->sm_hdr->samples[query_isample]);
}
}
case 't': targets = optarg; break;
case 'T': targets = optarg; targets_is_file = 1; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
if ( regions && bcf_sr_set_regions(args->files, regions, regions_is_file)<0 ) error("Failed to read the regions: %s\n", regions);
if ( targets && bcf_sr_set_targets(args->files, targets, targets_is_file, 0)<0 ) error("Failed to read the targets: %s\n", targets);
if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
- if ( args->gt_fname && !bcf_sr_add_reader(args->files, args->gt_fname) ) error("Failed to open %s: %s\n", args->gt_fname,bcf_sr_strerror(args->files->errnum));
+ if ( args->gt_fname && !bcf_sr_add_reader(args->files, args->gt_fname) )
+ error("Failed to read from %s: %s\n", !strcmp("-",args->gt_fname)?"standard input":args->gt_fname,bcf_sr_strerror(args->files->errnum));
args->files->collapse = COLLAPSE_SNPS|COLLAPSE_INDELS;
if ( args->plot ) args->plot = init_prefix(args->plot);
init_data(args);
int fake_PL = args->no_PLs ? args->no_PLs : 99; // with 1, discordance is the number of non-matching GTs
int nsm_gt, i;
if ( (nsm_gt=bcf_get_genotypes(hdr, line, &args->tmp_arr, &args->ntmp_arr)) <= 0 )
- error("GT not present at %s:%d?\n", hdr->id[BCF_DT_CTG][line->rid].key, line->pos+1);
+ error("GT not present at %s:%"PRId64"?\n", hdr->id[BCF_DT_CTG][line->rid].key, (int64_t) line->pos+1);
nsm_gt /= bcf_hdr_nsamples(hdr);
int npl = line->n_allele*(line->n_allele+1)/2;
hts_expand(int,npl*bcf_hdr_nsamples(hdr),args->npl_arr,args->pl_arr);
// Target genotypes
int ngt, npl;
if ( (ngt=bcf_get_genotypes(args->gt_hdr, gt_line, >_arr, &ngt_arr)) <= 0 )
- error("GT not present at %s:%d?", args->gt_hdr->id[BCF_DT_CTG][gt_line->rid].key, gt_line->pos+1);
+ error("GT not present at %s:%"PRId64"?", args->gt_hdr->id[BCF_DT_CTG][gt_line->rid].key, (int64_t) gt_line->pos+1);
ngt /= bcf_hdr_nsamples(args->gt_hdr);
if ( ngt!=2 ) continue; // checking only diploid genotypes
npl = fake_PLs(args, args->sm_hdr, sm_line);
}
else
- error("PL not present at %s:%d?\n", args->sm_hdr->id[BCF_DT_CTG][sm_line->rid].key, sm_line->pos+1);
+ error("PL not present at %s:%"PRId64"?\n", args->sm_hdr->id[BCF_DT_CTG][sm_line->rid].key, (int64_t) sm_line->pos+1);
}
else
npl /= bcf_hdr_nsamples(args->sm_hdr);
int a = bcf_gt_allele(gt_ptr[0]);
int b = bcf_gt_allele(gt_ptr[1]);
if ( args->hom_only && a!=b ) continue; // heterozygous genotype
- fprintf(fp, "SC\t%s\t%d", args->gt_hdr->id[BCF_DT_CTG][gt_line->rid].key, gt_line->pos+1);
+ fprintf(fp, "SC\t%s\t%"PRId64, args->gt_hdr->id[BCF_DT_CTG][gt_line->rid].key, (int64_t) gt_line->pos+1);
for (i=0; i<gt_line->n_allele; i++) fprintf(fp, "%c%s", i==0?'\t':',', gt_line->d.allele[i]);
fprintf(fp, "\t%s/%s", a>=0 ? gt_line->d.allele[a] : ".", b>=0 ? gt_line->d.allele[b] : ".");
fprintf(fp, "\t%f", args->lks[query_isample]-prev_lk);
if ( args->plot )
{
- fclose(fp);
+ if ( fclose(fp)!=0 ) error("[%s] Error: close failed\n", __func__);
plot_check(args, args->target_sample ? args->target_sample : "", args->sm_hdr->samples[query_isample]);
}
}
case 't': targets = optarg; break;
case 'T': targets = optarg; targets_is_file = 1; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
if ( regions && bcf_sr_set_regions(args->files, regions, regions_is_file)<0 ) error("Failed to read the regions: %s\n", regions);
if ( targets && bcf_sr_set_targets(args->files, targets, targets_is_file, 0)<0 ) error("Failed to read the targets: %s\n", targets);
if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
- if ( args->gt_fname && !bcf_sr_add_reader(args->files, args->gt_fname) ) error("Failed to open %s: %s\n", args->gt_fname,bcf_sr_strerror(args->files->errnum));
+ if ( args->gt_fname && !bcf_sr_add_reader(args->files, args->gt_fname) )
+ error("Failed to read from %s: %s\n", !strcmp("-",args->gt_fname)?"standard input":args->gt_fname,bcf_sr_strerror(args->files->errnum));
args->files->collapse = COLLAPSE_SNPS|COLLAPSE_INDELS;
if ( args->plot ) args->plot = init_prefix(args->plot);
init_data(args);
fprintf(stderr, " -m, --min-shift INT set minimal interval size for CSI indices to 2^INT [14]\n");
fprintf(stderr, " -o, --output-file FILE optional output index file name\n");
fprintf(stderr, " -t, --tbi generate TBI-format index for VCF files\n");
- fprintf(stderr, " --threads sets the number of threads [0]\n");
+ fprintf(stderr, " --threads INT use multithreading with INT worker threads [0]\n");
fprintf(stderr, "\n");
fprintf(stderr, "Stats options:\n");
fprintf(stderr, " -n, --nrecords print number of records based on existing index file\n");
}
if (stats&2) printf("%" PRIu64 "\n", sum);
free(seq);
- hts_close(fp);
+ if ( hts_close(fp)!=0 ) error("[%s] Error: close failed\n", __func__);
bcf_hdr_destroy(hdr);
if (tbx)
tbx_destroy(tbx);
fprintf(bcftools_stderr, " -m, --min-shift INT set minimal interval size for CSI indices to 2^INT [14]\n");
fprintf(bcftools_stderr, " -o, --output-file FILE optional output index file name\n");
fprintf(bcftools_stderr, " -t, --tbi generate TBI-format index for VCF files\n");
- fprintf(bcftools_stderr, " --threads sets the number of threads [0]\n");
+ fprintf(bcftools_stderr, " --threads INT use multithreading with INT worker threads [0]\n");
fprintf(bcftools_stderr, "\n");
fprintf(bcftools_stderr, "Stats options:\n");
fprintf(bcftools_stderr, " -n, --nrecords print number of records based on existing index file\n");
}
if (stats&2) fprintf(bcftools_stdout, "%" PRIu64 "\n", sum);
free(seq);
- hts_close(fp);
+ if ( hts_close(fp)!=0 ) error("[%s] Error: close failed\n", __func__);
bcf_hdr_destroy(hdr);
if (tbx)
tbx_destroy(tbx);
/* vcfisec.c -- Create intersections, unions and complements of VCF files.
- Copyright (C) 2012-2014 Genome Research Ltd.
+ Copyright (C) 2012-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/vcfutils.h>
+#include <htslib/hts_os.h>
#include "bcftools.h"
#include "filter.h"
if ( out_fh == NULL ) error("Can't write to %s: %s\n", args->output_fname? args->output_fname : "standard output", strerror(errno));
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
if (args->record_cmd_line) bcf_hdr_append_version(files->readers[args->iwrite].header,args->argc,args->argv,"bcftools_isec");
- bcf_hdr_write(out_fh, files->readers[args->iwrite].header);
+ if ( bcf_hdr_write(out_fh, files->readers[args->iwrite].header)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname?args->output_fname:"standard output");
}
if ( !args->nwrite && !out_std && !args->prefix )
fprintf(stderr,"Note: -w option not given, printing list of sites...\n");
if ( out_std )
{
- if ( bcf_sr_has_line(files,args->iwrite) )
- bcf_write1(out_fh, files->readers[args->iwrite].header, files->readers[args->iwrite].buffer[0]);
+ if ( bcf_sr_has_line(files,args->iwrite) && bcf_write1(out_fh, files->readers[args->iwrite].header, files->readers[args->iwrite].buffer[0])!=0 )
+ error("[%s] Error: cannot write to %s\n", __func__, args->output_fname ? args->output_fname : "standard output");
continue;
}
else if ( args->fh_sites )
for (i=0; i<files->nreaders; i++)
kputc(bcf_sr_has_line(files,i)?'1':'0', &str);
kputc('\n', &str);
- fwrite(str.s,sizeof(char),str.l,args->fh_sites);
+ if ( fwrite(str.s,sizeof(char),str.l,args->fh_sites)!=str.l )
+ error("[%s] Error: failed to write %d bytes to %s\n", __func__,(int)str.l,args->output_fname ? args->output_fname : "standard output");
}
if ( args->prefix )
if ( args->isec_op==OP_VENN && ret==3 )
{
if ( !args->nwrite || args->write[0] )
- bcf_write1(args->fh_out[2], bcf_sr_get_header(files,0), bcf_sr_get_line(files,0));
+ {
+ if ( bcf_write1(args->fh_out[2], bcf_sr_get_header(files,0), bcf_sr_get_line(files,0))!=0 )
+ error("[%s] Error: cannot write\n", __func__);
+ }
if ( !args->nwrite || args->write[1] )
- bcf_write1(args->fh_out[3], bcf_sr_get_header(files,1), bcf_sr_get_line(files,1));
+ {
+ if ( bcf_write1(args->fh_out[3], bcf_sr_get_header(files,1), bcf_sr_get_line(files,1))!=0 )
+ error("[%s] Error: cannot write\n", __func__);
+ }
}
else
{
{
if ( !bcf_sr_has_line(files,i) ) continue;
if ( args->write && !args->write[i] ) continue;
- bcf_write1(args->fh_out[i], files->readers[i].header, files->readers[i].buffer[0]);
+ if ( bcf_write1(args->fh_out[i], files->readers[i].header, files->readers[i].buffer[0])!=0 ) error("[%s] Error: cannot write\n", __func__);
}
}
}
}
if ( str.s ) free(str.s);
- if ( out_fh ) hts_close(out_fh);
+ if ( out_fh && hts_close(out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname? args->output_fname : "-");
}
static void add_filter(args_t *args, char *expr, int logic)
if ( !args->fh_out[i] ) error("Could not open %s\n", args->fnames[i]); \
if ( args->n_threads ) hts_set_threads(args->fh_out[i], args->n_threads); \
if (args->record_cmd_line) bcf_hdr_append_version(args->files->readers[j].header,args->argc,args->argv,"bcftools_isec"); \
- bcf_hdr_write(args->fh_out[i], args->files->readers[j].header); \
+ if ( bcf_hdr_write(args->fh_out[i], args->files->readers[j].header)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->fnames[i]); \
}
if ( !args->nwrite || args->write[0] )
{
for (i=0; i<n; i++)
{
if ( !args->fnames[i] ) continue;
- hts_close(args->fh_out[i]);
+ if ( hts_close(args->fh_out[i])!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fnames[i]);
if ( args->output_type==FT_VCF_GZ )
{
tbx_conf_t conf = tbx_conf_vcf;
fprintf(stderr, " -R, --regions-file <file> restrict to regions listed in a file\n");
fprintf(stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
- fprintf(stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(stderr, " -w, --write <list> list of files to write with -p given as 1-based indexes. By default, all files are written\n");
fprintf(stderr, "\n");
fprintf(stderr, "Examples:\n");
fprintf(stderr, " # Extract and write records from A shared by both A and B using exact allele match\n");
fprintf(stderr, " bcftools isec A.vcf.gz B.vcf.gz -p dir -n =2 -w 1\n");
fprintf(stderr, "\n");
+ fprintf(stderr, " # Extract and write records from C found in A and C but not in B\n");
+ fprintf(stderr, " bcftools isec A.vcf.gz B.vcf.gz C.vcf.gz -p dir -n~101 -w 3\n");
+ fprintf(stderr, "\n");
fprintf(stderr, " # Extract records private to A or B comparing by position only\n");
fprintf(stderr, " bcftools isec A.vcf.gz B.vcf.gz -p dir -n -1 -c all\n");
fprintf(stderr, "\n");
else error("The --collapse string \"%s\" not recognised.\n", optarg);
break;
case 'f': args->files->apply_filters = optarg; break;
- case 'C': args->isec_op = OP_COMPLEMENT; break;
+ case 'C':
+ if ( args->isec_op!=0 && args->isec_op!=OP_COMPLEMENT ) error("Error: either -C or -n should be given, not both.\n");
+ args->isec_op = OP_COMPLEMENT; break;
case 'r': args->regions_list = optarg; break;
case 'R': args->regions_list = optarg; regions_is_file = 1; break;
case 't': args->targets_list = optarg; break;
case 'e': add_filter(args, optarg, FLT_EXCLUDE); break;
case 'n':
{
+ if ( args->isec_op!=0 && args->isec_op==OP_COMPLEMENT ) error("Error: either -C or -n should be given, not both.\n");
+ if ( args->isec_op!=0 ) error("Error: -n should be given only once.\n");
char *p = optarg;
if ( *p=='-' ) { args->isec_op = OP_MINUS; p++; }
else if ( *p=='+' ) { args->isec_op = OP_PLUS; p++; }
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
/* vcfisec.c -- Create intersections, unions and complements of VCF files.
- Copyright (C) 2012-2014 Genome Research Ltd.
+ Copyright (C) 2012-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/vcfutils.h>
+#include <htslib/hts_os.h>
#include "bcftools.h"
#include "filter.h"
if ( out_fh == NULL ) error("Can't write to %s: %s\n", args->output_fname? args->output_fname : "standard output", strerror(errno));
if ( args->n_threads ) hts_set_threads(out_fh, args->n_threads);
if (args->record_cmd_line) bcf_hdr_append_version(files->readers[args->iwrite].header,args->argc,args->argv,"bcftools_isec");
- bcf_hdr_write(out_fh, files->readers[args->iwrite].header);
+ if ( bcf_hdr_write(out_fh, files->readers[args->iwrite].header)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname?args->output_fname:"standard output");
}
if ( !args->nwrite && !out_std && !args->prefix )
fprintf(bcftools_stderr,"Note: -w option not given, printing list of sites...\n");
if ( out_std )
{
- if ( bcf_sr_has_line(files,args->iwrite) )
- bcf_write1(out_fh, files->readers[args->iwrite].header, files->readers[args->iwrite].buffer[0]);
+ if ( bcf_sr_has_line(files,args->iwrite) && bcf_write1(out_fh, files->readers[args->iwrite].header, files->readers[args->iwrite].buffer[0])!=0 )
+ error("[%s] Error: cannot write to %s\n", __func__, args->output_fname ? args->output_fname : "standard output");
continue;
}
else if ( args->fh_sites )
for (i=0; i<files->nreaders; i++)
kputc(bcf_sr_has_line(files,i)?'1':'0', &str);
kputc('\n', &str);
- fwrite(str.s,sizeof(char),str.l,args->fh_sites);
+ if ( fwrite(str.s,sizeof(char),str.l,args->fh_sites)!=str.l )
+ error("[%s] Error: failed to write %d bytes to %s\n", __func__,(int)str.l,args->output_fname ? args->output_fname : "standard output");
}
if ( args->prefix )
if ( args->isec_op==OP_VENN && ret==3 )
{
if ( !args->nwrite || args->write[0] )
- bcf_write1(args->fh_out[2], bcf_sr_get_header(files,0), bcf_sr_get_line(files,0));
+ {
+ if ( bcf_write1(args->fh_out[2], bcf_sr_get_header(files,0), bcf_sr_get_line(files,0))!=0 )
+ error("[%s] Error: cannot write\n", __func__);
+ }
if ( !args->nwrite || args->write[1] )
- bcf_write1(args->fh_out[3], bcf_sr_get_header(files,1), bcf_sr_get_line(files,1));
+ {
+ if ( bcf_write1(args->fh_out[3], bcf_sr_get_header(files,1), bcf_sr_get_line(files,1))!=0 )
+ error("[%s] Error: cannot write\n", __func__);
+ }
}
else
{
{
if ( !bcf_sr_has_line(files,i) ) continue;
if ( args->write && !args->write[i] ) continue;
- bcf_write1(args->fh_out[i], files->readers[i].header, files->readers[i].buffer[0]);
+ if ( bcf_write1(args->fh_out[i], files->readers[i].header, files->readers[i].buffer[0])!=0 ) error("[%s] Error: cannot write\n", __func__);
}
}
}
}
if ( str.s ) free(str.s);
- if ( out_fh ) hts_close(out_fh);
+ if ( out_fh && hts_close(out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname? args->output_fname : "-");
}
static void add_filter(args_t *args, char *expr, int logic)
if ( !args->fh_out[i] ) error("Could not open %s\n", args->fnames[i]); \
if ( args->n_threads ) hts_set_threads(args->fh_out[i], args->n_threads); \
if (args->record_cmd_line) bcf_hdr_append_version(args->files->readers[j].header,args->argc,args->argv,"bcftools_isec"); \
- bcf_hdr_write(args->fh_out[i], args->files->readers[j].header); \
+ if ( bcf_hdr_write(args->fh_out[i], args->files->readers[j].header)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->fnames[i]); \
}
if ( !args->nwrite || args->write[0] )
{
for (i=0; i<n; i++)
{
if ( !args->fnames[i] ) continue;
- hts_close(args->fh_out[i]);
+ if ( hts_close(args->fh_out[i])!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fnames[i]);
if ( args->output_type==FT_VCF_GZ )
{
tbx_conf_t conf = tbx_conf_vcf;
fprintf(bcftools_stderr, " -R, --regions-file <file> restrict to regions listed in a file\n");
fprintf(bcftools_stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(bcftools_stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(bcftools_stderr, " -w, --write <list> list of files to write with -p given as 1-based indexes. By default, all files are written\n");
fprintf(bcftools_stderr, "\n");
fprintf(bcftools_stderr, "Examples:\n");
fprintf(bcftools_stderr, " # Extract and write records from A shared by both A and B using exact allele match\n");
fprintf(bcftools_stderr, " bcftools isec A.vcf.gz B.vcf.gz -p dir -n =2 -w 1\n");
fprintf(bcftools_stderr, "\n");
+ fprintf(bcftools_stderr, " # Extract and write records from C found in A and C but not in B\n");
+ fprintf(bcftools_stderr, " bcftools isec A.vcf.gz B.vcf.gz C.vcf.gz -p dir -n~101 -w 3\n");
+ fprintf(bcftools_stderr, "\n");
fprintf(bcftools_stderr, " # Extract records private to A or B comparing by position only\n");
fprintf(bcftools_stderr, " bcftools isec A.vcf.gz B.vcf.gz -p dir -n -1 -c all\n");
fprintf(bcftools_stderr, "\n");
else error("The --collapse string \"%s\" not recognised.\n", optarg);
break;
case 'f': args->files->apply_filters = optarg; break;
- case 'C': args->isec_op = OP_COMPLEMENT; break;
+ case 'C':
+ if ( args->isec_op!=0 && args->isec_op!=OP_COMPLEMENT ) error("Error: either -C or -n should be given, not both.\n");
+ args->isec_op = OP_COMPLEMENT; break;
case 'r': args->regions_list = optarg; break;
case 'R': args->regions_list = optarg; regions_is_file = 1; break;
case 't': args->targets_list = optarg; break;
case 'e': add_filter(args, optarg, FLT_EXCLUDE); break;
case 'n':
{
+ if ( args->isec_op!=0 && args->isec_op==OP_COMPLEMENT ) error("Error: either -C or -n should be given, not both.\n");
+ if ( args->isec_op!=0 ) error("Error: -n should be given only once.\n");
char *p = optarg;
if ( *p=='-' ) { args->isec_op = OP_MINUS; p++; }
else if ( *p=='+' ) { args->isec_op = OP_PLUS; p++; }
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
/* vcfmerge.c -- Merge multiple VCF/BCF files to create one multi-sample file.
- Copyright (C) 2012-2016 Genome Research Ltd.
+ Copyright (C) 2012-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#include <errno.h>
#include <unistd.h>
#include <getopt.h>
+#include <inttypes.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/vcfutils.h>
typedef struct
{
bcf1_t *line;
- int end, active;
+ int end, active; // end: 0-based INFO/END
}
gvcf_aux_t;
int nfmt_map; // number of rows in the fmt_map array
int *agr_map, nagr_map, magr_map; // mapping between Number=AGR element indexes
void *tmp_arr;
- int ntmp_arr;
+ size_t ntmp_arr;
buffer_t *buf;
AGR_info_t *AGR_info;
int nAGR_info, mAGR_info;
bcf_srs_t *files;
- int gvcf_min, gvcf_break; // min buffered gvcf END position (NB: gvcf_min is 1-based) or 0 if no active lines are present
- gvcf_aux_t *gvcf; // buffer of gVCF lines
+ int gvcf_min, // min buffered gvcf END position (NB: gvcf_min is 1-based) or 0 if no active lines are present
+ gvcf_break; // 0-based position of a next record which breaks a gVCF block
+ gvcf_aux_t *gvcf; // buffer of gVCF lines, for each reader one line
+ int nout_smpl;
+ kstring_t *str;
}
maux_t;
{
int msize = args->maux->ntmp_arr / rule->type_size;
int ret = bcf_get_info_values(hdr, line, rule->hdr_tag, &args->maux->tmp_arr, &msize, rule->type);
- if ( ret<=0 ) error("FIXME: error parsing %s at %s:%d .. %d\n", rule->hdr_tag,bcf_seqname(hdr,line),line->pos+1,ret);
+ if ( ret<=0 ) error("FIXME: error parsing %s at %s:%"PRId64" .. %d\n", rule->hdr_tag,bcf_seqname(hdr,line),(int64_t) line->pos+1,ret);
args->maux->ntmp_arr = msize * rule->type_size;
rule->nblocks++;
int i, j;
if ( var_len==BCF_VL_A )
{
- if ( ret!=line->n_allele-1 ) error("Wrong number of %s fields at %s:%d\n",rule->hdr_tag,bcf_seqname(hdr,line),line->pos+1);
+ if ( ret!=line->n_allele-1 ) error("Wrong number of %s fields at %s:%"PRId64"\n",rule->hdr_tag,bcf_seqname(hdr,line),(int64_t) line->pos+1);
args->maux->nagr_map = ret;
hts_expand(int,args->maux->nagr_map,args->maux->magr_map,args->maux->agr_map);
// create mapping from source file ALT indexes to dst file indexes
}
else if ( var_len==BCF_VL_R )
{
- if ( ret!=line->n_allele ) error("Wrong number of %s fields at %s:%d\n",rule->hdr_tag,bcf_seqname(hdr,line),line->pos+1);
+ if ( ret!=line->n_allele ) error("Wrong number of %s fields at %s:%"PRId64"\n",rule->hdr_tag,bcf_seqname(hdr,line),(int64_t) line->pos+1);
args->maux->nagr_map = ret;
hts_expand(int,args->maux->nagr_map,args->maux->magr_map,args->maux->agr_map);
for (i=0; i<ret; i++) args->maux->agr_map[i] = als->map[i];
else
{
if ( rule->nblocks>1 && ret!=rule->block_size )
- error("Mismatch in number of values for INFO/%s at %s:%d\n", rule->hdr_tag,bcf_seqname(hdr,line),line->pos+1);
+ error("Mismatch in number of values for INFO/%s at %s:%"PRId64"\n", rule->hdr_tag,bcf_seqname(hdr,line),(int64_t) line->pos+1);
rule->block_size = ret;
args->maux->nagr_map = 0;
}
int i;
for (i=0; i<bcf_hdr_nsamples(hr); i++)
{
- char *name = hr->samples[i];
- if ( bcf_hdr_id2int(hw, BCF_DT_SAMPLE, name)!=-1 )
+ char *rmme = NULL, *name = hr->samples[i];
+ while ( bcf_hdr_id2int(hw, BCF_DT_SAMPLE, name)!=-1 )
{
// there is a sample with the same name
if ( !force_samples ) error("Error: Duplicate sample names (%s), use --force-samples to proceed anyway.\n", name);
- int len = strlen(hr->samples[i]) + strlen(clash_prefix) + 1;
- name = (char*) malloc(sizeof(char)*(len+1));
- sprintf(name,"%s:%s",clash_prefix,hr->samples[i]);
- bcf_hdr_add_sample(hw,name);
- free(name);
+ // Resolve conflicting samples names. For example, replace:
+ // A + A with A,2:A
+ // A,2:A + A with A,2:A,2:2:A
+
+ int len = strlen(name) + strlen(clash_prefix) + 1;
+ char *tmp = (char*) malloc(sizeof(char)*(len+1));
+ sprintf(tmp,"%s:%s",clash_prefix,name);
+ free(rmme);
+ rmme = name = tmp;
}
- else
- bcf_hdr_add_sample(hw,name);
+ bcf_hdr_add_sample(hw,name);
+ free(rmme);
}
}
int i, n_smpl = 0;
for (i=0; i<ma->n; i++)
n_smpl += bcf_hdr_nsamples(files->readers[i].header);
+ ma->nout_smpl = n_smpl;
+ assert( n_smpl==bcf_hdr_nsamples(args->out_hdr) );
if ( args->do_gvcf )
{
ma->gvcf = (gvcf_aux_t*) calloc(ma->n,sizeof(gvcf_aux_t));
ma->buf = (buffer_t*) calloc(ma->n,sizeof(buffer_t));
for (i=0; i<ma->n; i++)
ma->buf[i].rid = -1;
+ ma->str = (kstring_t*) calloc(n_smpl,sizeof(kstring_t));
return ma;
}
void maux_destroy(maux_t *ma)
{
int i,j;
+ for (i=0; i<ma->nout_smpl; i++) free(ma->str[i].s);
+ free(ma->str);
for (i=0; i<ma->mals; i++)
{
free(ma->als[i]);
}
ma->buf[i].end = j;
ma->buf[i].cur = -1;
- if ( ma->buf[i].beg < ma->buf[i].end )
+ if ( ma->buf[i].beg < ma->buf[i].end )
{
ma->buf[i].lines = ma->files->readers[i].buffer;
if ( ma->gvcf ) ma->gvcf[i].active = 0; // gvcf block cannot overlap with the next record
int end_src = start_src;
while ( end_src<src_len && src[end_src] && src[end_src]!=',' ) end_src++;
- int nsrc_cpy = end_src - start_src;
+ int nsrc_cpy = end_src - start_src; // number of chars to copy (excluding \0)
if ( nsrc_cpy==1 && src[start_src]=='.' ) return 0; // don't write missing values, dst is already initialized
int ith_dst = 0, start_dst = 0;
agr->mbuf = tmp.m; agr->nbuf = tmp.l; agr->buf = (uint8_t*)tmp.s;
}
else
- error("Not ready for type [%d]: %s at %d\n", info->type,agr->hdr_tag,line->pos+1);
+ error("Not ready for type [%d]: %s at %"PRId64"\n", info->type,agr->hdr_tag,(int64_t) line->pos+1);
}
if ( info->type==BCF_BT_INT8 || info->type==BCF_BT_INT16 || info->type==BCF_BT_INT32 || info->type==BCF_BT_FLOAT )
{
int ret = copy_string_field((char*)info->vptr, iori-ifrom, info->len, &tmp, als->map[iori]-ifrom);
if ( ret )
- error("Error at %s:%d: wrong number of fields in %s?\n", bcf_seqname(hdr,line),line->pos+1,agr->hdr_tag);
+ error("Error at %s:%"PRId64": wrong number of fields in %s?\n", bcf_seqname(hdr,line),(int64_t) line->pos+1,agr->hdr_tag);
}
}
else
int knew = bcf_alleles2gt(inew,jnew);
int ret = copy_string_field((char*)info->vptr, kori, info->len, &tmp, knew);
if ( ret )
- error("Error at %s:%d: wrong number of fields in %s?\n", bcf_seqname(hdr,line),line->pos+1,agr->hdr_tag);
+ error("Error at %s:%"PRId64": wrong number of fields in %s?\n", bcf_seqname(hdr,line),(int64_t) line->pos+1,agr->hdr_tag);
}
}
}
}
kitr = kh_get(strdict, tmph, key);
int idx = kh_val(tmph, kitr);
- if ( idx<0 ) error("Error occurred while processing INFO tag \"%s\" at %s:%d\n", key,bcf_seqname(hdr,line),line->pos+1);
+ if ( idx<0 ) error("Error occurred while processing INFO tag \"%s\" at %s:%"PRId64"\n", key,bcf_seqname(hdr,line),(int64_t) line->pos+1);
merge_AGR_info_tag(hdr, line,inf,len,&ma->buf[i].rec[irec],&ma->AGR_info[idx]);
continue;
}
bcf_hdr_t *out_hdr = args->out_hdr;
maux_t *ma = args->maux;
int i, ismpl = 0, nsamples = bcf_hdr_nsamples(out_hdr);
+ static int warned = 0;
int nsize = 0, msize = sizeof(int32_t);
for (i=0; i<files->nreaders; i++)
{
ma->ntmp_arr = nsamples*nsize*msize;
ma->tmp_arr = realloc(ma->tmp_arr, ma->ntmp_arr);
+ if ( !ma->tmp_arr ) error("Could not allocate %zu bytes\n",ma->ntmp_arr);
+ if ( ma->ntmp_arr > 2147483647 )
+ {
+ if ( !warned ) fprintf(stderr,"Warning: Too many genotypes at %s:%"PRId64", requires %zu bytes, skipping.\n", bcf_seqname(out_hdr,out),(int64_t) out->pos+1,ma->ntmp_arr);
+ warned = 1;
+ return;
+ }
}
memset(ma->smpl_ploidy,0,nsamples*sizeof(int));
bcf_update_format_int32(out_hdr, out, "GT", (int32_t*)ma->tmp_arr, nsamples*nsize);
}
+void merge_format_string(args_t *args, const char *key, bcf_fmt_t **fmt_map, bcf1_t *out, int length, int nsize)
+{
+ bcf_srs_t *files = args->files;
+ bcf_hdr_t *out_hdr = args->out_hdr;
+ maux_t *ma = args->maux;
+ int i,j, nsamples = bcf_hdr_nsamples(out_hdr);
+ static int warned = 0;
+
+ // initialize empty strings, a dot for each value, e.g. ".,.,."
+ int nmax = 0;
+ for (i=0; i<nsamples; i++)
+ {
+ kstring_t *str = &ma->str[i];
+ if ( length==BCF_VL_FIXED || length==BCF_VL_VAR )
+ {
+ str->l = 1;
+ ks_resize(str, str->l+1);
+ str->s[0] = '.';
+ }
+ else
+ {
+ str->l = nsize*2 - 1;
+ ks_resize(str, str->l+1);
+ str->s[0] = '.';
+ for (j=1; j<nsize; j++) str->s[j*2-1] = ',', str->s[j*2] = '.';
+ }
+ str->s[str->l] = 0;
+ if ( nmax < str->l ) nmax = str->l;
+ }
+
+ // fill in values for each sample
+ int ismpl = 0;
+ for (i=0; i<files->nreaders; i++)
+ {
+ bcf_sr_t *reader = &files->readers[i];
+ bcf_hdr_t *hdr = reader->header;
+ bcf_fmt_t *fmt_ori = fmt_map[i];
+ if ( !fmt_ori )
+ {
+ // the field is not present in this file
+ ismpl += bcf_hdr_nsamples(hdr);
+ continue;
+ }
+
+ bcf1_t *line = maux_get_line(args, i);
+ int irec = ma->buf[i].cur;
+ char *src = (char*) fmt_ori->p;
+
+ if ( length==BCF_VL_FIXED || length==BCF_VL_VAR || (line->n_allele==out->n_allele && !ma->buf[i].rec[irec].als_differ) )
+ {
+ // alleles unchanged, copy over
+ for (j=0; j<bcf_hdr_nsamples(hdr); j++)
+ {
+ kstring_t *str = &ma->str[ismpl++];
+ str->l = 0;
+ kputsn(src, fmt_ori->n, str);
+ if ( nmax < str->l ) nmax = str->l;
+ src += fmt_ori->n;
+ }
+ continue;
+ }
+ // NB, what is below is not the fastest way, copy_string_field() keeps
+ // finding the indexes repeatedly at multiallelic sites
+ if ( length==BCF_VL_A || length==BCF_VL_R )
+ {
+ int ifrom = length==BCF_VL_A ? 1 : 0;
+ for (j=0; j<bcf_hdr_nsamples(hdr); j++)
+ {
+ kstring_t *str = &ma->str[ismpl++];
+ int iori,inew;
+ for (iori=ifrom; iori<line->n_allele; iori++)
+ {
+ inew = ma->buf[i].rec[irec].map[iori] - ifrom;
+ int ret = copy_string_field(src, iori - ifrom, fmt_ori->size, str, inew);
+ if ( ret<-1 ) error("[E::%s] fixme: internal error at %s:%"PRId64" .. %d\n",__func__,bcf_seqname(hdr,line),(int64_t) line->pos+1,ret);
+ }
+ src += fmt_ori->size;
+ }
+ continue;
+ }
+ assert( length==BCF_VL_G );
+ error("[E::%s] Merging of Number=G FORMAT strings (in your case FORMAT/%s) is not supported yet, sorry!\n"
+ "Please open an issue on github if this feature is essential for you. However, note that using FORMAT strings is not\n"
+ "a good idea in general - it is slow to parse and does not compress well, it is better to use integer codes instead.\n"
+ "If you don't really need it, use `bcftools annotate -x` to remove the annotation before merging.\n", __func__,key);
+ }
+ // update the record
+ if ( ma->ntmp_arr < nsamples*nmax )
+ {
+ ma->ntmp_arr = nsamples*nmax;
+ ma->tmp_arr = realloc(ma->tmp_arr, ma->ntmp_arr);
+ if ( !ma->tmp_arr ) error("Could not allocate %zu bytes\n",ma->ntmp_arr);
+ if ( ma->ntmp_arr > 2147483647 )
+ {
+ if ( !warned ) fprintf(stderr,"Warning: The row size is too big for FORMAT/%s at %s:%"PRId64", requires %zu bytes, skipping.\n", key,bcf_seqname(out_hdr,out),(int64_t) out->pos+1,ma->ntmp_arr);
+ warned = 1;
+ return;
+ }
+ }
+ char *tgt = (char*) ma->tmp_arr;
+ for (i=0; i<nsamples; i++)
+ {
+ memcpy(tgt, ma->str[i].s, ma->str[i].l);
+ if ( ma->str[i].l < nmax ) memset(tgt + ma->str[i].l, 0, nmax - ma->str[i].l);
+ tgt += nmax;
+ }
+ bcf_update_format_char(out_hdr, out, key, (float*)ma->tmp_arr, nsamples*nmax);
+}
+
void merge_format_field(args_t *args, bcf_fmt_t **fmt_map, bcf1_t *out)
{
bcf_srs_t *files = args->files;
bcf_hdr_t *out_hdr = args->out_hdr;
maux_t *ma = args->maux;
int i, ismpl = 0, nsamples = bcf_hdr_nsamples(out_hdr);
+ static int warned = 0;
const char *key = NULL;
- int nsize = 0, length = BCF_VL_FIXED, type = -1;
+ size_t nsize = 0, length = BCF_VL_FIXED;
+ int type = -1;
for (i=0; i<files->nreaders; i++)
{
if ( !maux_get_line(args,i) ) continue;
}
if ( fmt_map[i]->n > nsize ) nsize = fmt_map[i]->n;
}
+ if ( type==BCF_BT_CHAR )
+ {
+ merge_format_string(args, key, fmt_map, out, length, nsize);
+ return;
+ }
- int msize = sizeof(float)>sizeof(int32_t) ? sizeof(float) : sizeof(int32_t);
+ size_t msize = sizeof(float)>sizeof(int32_t) ? sizeof(float) : sizeof(int32_t);
if ( ma->ntmp_arr < nsamples*nsize*msize )
{
ma->ntmp_arr = nsamples*nsize*msize;
ma->tmp_arr = realloc(ma->tmp_arr, ma->ntmp_arr);
+ if ( !ma->tmp_arr ) error("Failed to allocate %zu bytes at %s:%"PRId64" for FORMAT/%s\n", ma->ntmp_arr,bcf_seqname(args->out_hdr,out),(int64_t) out->pos+1,key);
+ if ( ma->ntmp_arr > 2147483647 )
+ {
+ if ( !warned ) fprintf(stderr,"Warning: The row size is too big for FORMAT/%s at %s:%"PRId64", requires %zu bytes, skipping.\n", key,bcf_seqname(out_hdr,out),(int64_t) out->pos+1,ma->ntmp_arr);
+ warned = 1;
+ return;
+ }
}
// Fill the temp array for all samples by collecting values from all files
bcf_fmt_t *fmt_ori = fmt_map[i];
bcf1_t *line = maux_get_line(args, i);
int irec = ma->buf[i].cur;
+
if ( fmt_ori )
{
type = fmt_ori->type;
{
// if all fields are missing then n==1 is valid
if ( fmt_ori->n!=1 && fmt_ori->n != nals_ori*(nals_ori+1)/2 && fmt_map[i]->n != nals_ori )
- error("Incorrect number of FORMAT/%s values at %s:%d, cannot merge. The tag is defined as Number=G, but found\n"
+ error("Incorrect number of FORMAT/%s values at %s:%"PRId64", cannot merge. The tag is defined as Number=G, but found\n"
"%d values and %d alleles. See also http://samtools.github.io/bcftools/howtos/FAQ.html#incorrect-nfields\n",
- key,bcf_seqname(args->out_hdr,out),out->pos+1,fmt_ori->n,nals_ori);
+ key,bcf_seqname(args->out_hdr,out),(int64_t) out->pos+1,fmt_ori->n,nals_ori);
}
else if ( length==BCF_VL_A )
{
if ( fmt_ori->n!=1 && fmt_ori->n != nals_ori-1 )
- error("Incorrect number of FORMAT/%s values at %s:%d, cannot merge. The tag is defined as Number=A, but found\n"
+ error("Incorrect number of FORMAT/%s values at %s:%"PRId64", cannot merge. The tag is defined as Number=A, but found\n"
"%d values and %d alleles. See also http://samtools.github.io/bcftools/howtos/FAQ.html#incorrect-nfields\n",
- key,bcf_seqname(args->out_hdr,out),out->pos+1,fmt_ori->n,nals_ori);
+ key,bcf_seqname(args->out_hdr,out),(int64_t) out->pos+1,fmt_ori->n,nals_ori);
}
else if ( length==BCF_VL_R )
{
if ( fmt_ori->n!=1 && fmt_ori->n != nals_ori )
- error("Incorrect number of FORMAT/%s values at %s:%d, cannot merge. The tag is defined as Number=R, but found\n"
+ error("Incorrect number of FORMAT/%s values at %s:%"PRId64", cannot merge. The tag is defined as Number=R, but found\n"
"%d values and %d alleles. See also http://samtools.github.io/bcftools/howtos/FAQ.html#incorrect-nfields\n",
- key,bcf_seqname(args->out_hdr,out),out->pos+1,fmt_ori->n,nals_ori);
+ key,bcf_seqname(args->out_hdr,out),(int64_t) out->pos+1,fmt_ori->n,nals_ori);
}
}
case BCF_BT_INT16: BRANCH(int32_t, int16_t, *src==bcf_int16_missing, *src==bcf_int16_vector_end, *tgt=bcf_int32_missing, *tgt=bcf_int32_vector_end); break;
case BCF_BT_INT32: BRANCH(int32_t, int32_t, *src==bcf_int32_missing, *src==bcf_int32_vector_end, *tgt=bcf_int32_missing, *tgt=bcf_int32_vector_end); break;
case BCF_BT_FLOAT: BRANCH(float, float, bcf_float_is_missing(*src), bcf_float_is_vector_end(*src), bcf_float_set_missing(*tgt), bcf_float_set_vector_end(*tgt)); break;
- case BCF_BT_CHAR: BRANCH(uint8_t, uint8_t, *src==bcf_str_missing, *src==bcf_str_vector_end, *tgt=bcf_str_missing, *tgt=bcf_str_vector_end); break;
default: error("Unexpected case: %d, %s\n", type, key);
}
#undef BRANCH
}
if ( type==BCF_BT_FLOAT )
bcf_update_format_float(out_hdr, out, key, (float*)ma->tmp_arr, nsamples*nsize);
- else if ( type==BCF_BT_CHAR )
- bcf_update_format_char(out_hdr, out, key, (float*)ma->tmp_arr, nsamples*nsize);
else
bcf_update_format_int32(out_hdr, out, key, (int32_t*)ma->tmp_arr, nsamples*nsize);
}
{
if ( !gaux[i].active ) continue;
bcf1_t *line = maux_get_line(args, i);
+ if ( !line ) continue;
int irec = maux->buf[i].cur;
hts_expand(int, line->n_allele, maux->buf[i].rec[irec].mmap, maux->buf[i].rec[irec].map);
if ( !maux->als )
{
bcf_hdr_t *hdr = bcf_sr_get_header(args->files,i);
- error("Failed to merge alleles at %s:%d\n",bcf_seqname(hdr,line),line->pos+1);
+ error("Failed to merge alleles at %s:%"PRId64"\n",bcf_seqname(hdr,line),(int64_t) line->pos+1);
}
}
}
/*
Output staged gVCF blocks, end is the last position of the block. Assuming
gaux[i].active flags are set and maux_get_line returns correct lines.
+ Both start,end coordinates are 0-based.
*/
void gvcf_write_block(args_t *args, int start, int end)
{
assert(gaux);
// Update POS
- int min = INT_MAX;
+ int min = INT_MAX; // the minimum active gVCF INFO/END (0-based)
char ref = 'N';
for (i=0; i<args->files->nreaders; i++)
{
if ( min > gaux[i].end ) min = gaux[i].end;
}
// Check for valid gVCF blocks in this region
- if ( min==INT_MAX )
+ if ( min==INT_MAX ) // this probably should not happen
{
assert(0);
maux->gvcf_min = 0;
}
else
bcf_update_info_int32(args->out_hdr, out, "END", NULL, 0);
- bcf_write1(args->out_fh, args->out_hdr, out);
+ if ( bcf_write1(args->out_fh, args->out_hdr, out)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
bcf_clear1(out);
}
// When called on a region, trim the blocks accordingly
- int start = maux->gvcf_break>=0 ? maux->gvcf_break + 1 : maux->pos;
+ int start = maux->gvcf_break>=0 ? maux->gvcf_break + 1 : maux->pos; // the start of a new gvcf block to output
if ( args->regs )
{
int rstart = -1, rend = -1;
// does the block end before the new line or is it interrupted?
int tmp = maux->gvcf_min < flush_until ? maux->gvcf_min : flush_until;
if ( start > tmp-1 ) break;
- gvcf_write_block(args,start,tmp-1); // gvcf_min is 1-based
+ gvcf_write_block(args,start,tmp-1); // gvcf_min is 1-based, passing 0-based coordinates
start = tmp;
}
}
Check incoming lines for new gVCF blocks, set pointer to the current source
buffer (gvcf or readers). In contrast to gvcf_flush, this function can be
called only after maux_reset as it relies on updated maux buffers.
+ The coordinate is 0-based
*/
void gvcf_stage(args_t *args, int pos)
{
int ret = bcf_get_info_int32(hdr,line,"END",&end,&nend);
if ( ret==1 )
{
+ if ( end[0] == line->pos + 1 ) // POS and INFO/END are identical, treat as if a normal w/o INFO/END
+ {
+ maux->gvcf_break = line->pos;
+ continue;
+ }
+ if ( end[0] <= line->pos ) error("Error: Incorrect END at %s:%"PRId64" .. END=%d\n", bcf_seqname(hdr,line),(int64_t) line->pos+1,end[0]);
+
// END is set, this is a new gVCF block. Cache this line in gaux[i] and swap with
// an empty record: the gaux line must be kept until we reach its END.
+
gaux[i].active = 1;
gaux[i].end = end[0] - 1;
SWAP(bcf1_t*,args->files->readers[i].buffer[irec],gaux[i].line);
{
// Invalidate pointer to reader's buffer or else gvcf_flush will attempt
// to use the old lines via maux_get_line()
- if ( ma->gvcf && !ma->gvcf[ir].active ) ma->buf[ir].cur = -1;
+ if ( ma->gvcf )
+ {
+ if ( ma->gvcf[ir].active )
+ {
+ if ( ma->pos >= ma->gvcf[ir].end ) ma->gvcf[ir].active = 0;
+ else if ( ma->buf[ir].cur==-1 ) ma->buf[ir].cur = ma->buf[ir].beg; // re-activate interrupted gVCF block
+ }
+ if ( !ma->gvcf[ir].active ) ma->buf[ir].cur = -1;
+ }
bcf_sr_t *reader = bcf_sr_get_reader(args->files,ir);
if ( !reader->nbuffer ) continue; // nothing to clean
bcf_hdr_t *hdr = bcf_sr_get_header(args->files,i);
const char *chr = bcf_hdr_id2name(hdr, maux->buf[i].rid);
fprintf(stderr,"\t");
- for (j=maux->buf[i].beg; j<maux->buf[i].end; j++) fprintf(stderr," %s:%d",chr,maux->buf[i].lines[j]->pos+1);
+ for (j=maux->buf[i].beg; j<maux->buf[i].end; j++) fprintf(stderr," %s:%"PRId64,chr,(int64_t) maux->buf[i].lines[j]->pos+1);
}
fprintf(stderr,"\n");
}
+ fprintf(stderr,"gvcf_min=%d\n", args->maux->gvcf_min);
for (i=0; i<args->files->nreaders; i++)
{
fprintf(stderr,"reader %d:\tgvcf_active=%d", i,maux->gvcf[i].active);
- if ( maux->gvcf[i].active ) fprintf(stderr,"\tpos,end=%d,%d", maux->gvcf[i].line->pos+1,maux->gvcf[i].end+1);
+ if ( maux->gvcf[i].active ) fprintf(stderr,"\tpos,end=%"PRId64",%"PRId64, (int64_t) maux->gvcf[i].line->pos+1,(int64_t) maux->gvcf[i].end+1);
fprintf(stderr,"\n");
}
fprintf(stderr,"\n");
}
// normalize alleles
maux->als = merge_alleles(line->d.allele, line->n_allele, buf->rec[j].map, maux->als, &maux->nals, &maux->mals);
- if ( !maux->als ) error("Failed to merge alleles at %s:%d in %s\n",maux->chr,line->pos+1,reader->fname);
+ if ( !maux->als ) error("Failed to merge alleles at %s:%"PRId64" in %s\n",maux->chr,(int64_t) line->pos+1,reader->fname);
hts_expand0(int, maux->nals, maux->ncnt, maux->cnt);
for (k=1; k<line->n_allele; k++)
maux->cnt[ buf->rec[j].map[k] ]++; // how many times an allele appears in the files
if ( args->do_gvcf )
bcf_update_info_int32(args->out_hdr, out, "END", NULL, 0);
merge_format(args, out);
- bcf_write1(args->out_fh, args->out_hdr, out);
+ if ( bcf_write1(args->out_fh, args->out_hdr, out)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
bcf_clear1(out);
}
void bcf_hdr_append_version(bcf_hdr_t *hdr, int argc, char **argv, const char *cmd)
{
kstring_t str = {0,0,0};
- ksprintf(&str,"##%sVersion=%s+htslib-%s\n", cmd, bcftools_version(), hts_version());
- bcf_hdr_append(hdr,str.s);
+ int e = 0;
+ if (ksprintf(&str,"##%sVersion=%s+htslib-%s\n", cmd, bcftools_version(), hts_version()) < 0)
+ goto fail;
+ if (bcf_hdr_append(hdr,str.s) < 0)
+ goto fail;
str.l = 0;
- ksprintf(&str,"##%sCommand=%s", cmd, argv[0]);
+ e |= ksprintf(&str,"##%sCommand=%s", cmd, argv[0]) < 0;
int i;
for (i=1; i<argc; i++)
{
if ( strchr(argv[i],' ') )
- ksprintf(&str, " '%s'", argv[i]);
+ e |= ksprintf(&str, " '%s'", argv[i]) < 0;
else
- ksprintf(&str, " %s", argv[i]);
- }
- kputs("; Date=", &str);
- time_t tm; time(&tm); kputs(ctime(&tm), &str);
- kputc('\n', &str);
- bcf_hdr_append(hdr,str.s);
+ e |= ksprintf(&str, " %s", argv[i]) < 0;
+ }
+ e |= kputs("; Date=", &str) < 0;
+ time_t tm; time(&tm);
+ e |= kputs(ctime(&tm), &str) < 0;
+ e |= kputc('\n', &str) < 0;
+ if (e)
+ goto fail;
+ if (bcf_hdr_append(hdr,str.s) < 0)
+ goto fail;
+ free(ks_release(&str));
+
+ if (bcf_hdr_sync(hdr) < 0)
+ goto fail;
+ return;
+
+ fail:
free(str.s);
-
- bcf_hdr_sync(hdr);
+ error_errno("[%s] Failed to add program information to header", __func__);
}
void merge_vcf(args_t *args)
int i;
for (i=0; i<args->files->nreaders; i++)
{
- char buf[10]; snprintf(buf,10,"%d",i+1);
+ char buf[24]; snprintf(buf,sizeof buf,"%d",i+1);
merge_headers(args->out_hdr, args->files->readers[i].header,buf,args->force_samples);
}
if (args->record_cmd_line) bcf_hdr_append_version(args->out_hdr, args->argc, args->argv, "bcftools_merge");
- bcf_hdr_sync(args->out_hdr);
+ if (bcf_hdr_sync(args->out_hdr) < 0)
+ error_errno("[%s] Failed to update header", __func__);
}
info_rules_init(args);
bcf_hdr_set_version(args->out_hdr, bcf_hdr_get_version(args->files->readers[0].header));
- bcf_hdr_write(args->out_fh, args->out_hdr);
+ if ( bcf_hdr_write(args->out_fh, args->out_hdr)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
if ( args->header_only )
{
bcf_hdr_destroy(args->out_hdr);
- hts_close(args->out_fh);
+ if ( hts_close(args->out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
return;
}
info_rules_destroy(args);
maux_destroy(args->maux);
bcf_hdr_destroy(args->out_hdr);
- hts_close(args->out_fh);
+ if ( hts_close(args->out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
bcf_destroy1(args->out_line);
kh_destroy(strdict, args->tmph);
if ( args->tmps.m ) free(args->tmps.s);
fprintf(stderr, " -O, --output-type <b|u|z|v> 'b' compressed BCF; 'u' uncompressed BCF; 'z' compressed VCF; 'v' uncompressed VCF [v]\n");
fprintf(stderr, " -r, --regions <region> restrict to comma-separated list of regions\n");
fprintf(stderr, " -R, --regions-file <file> restrict to regions listed in a file\n");
- fprintf(stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(stderr, "\n");
exit(1);
}
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
/* vcfmerge.c -- Merge multiple VCF/BCF files to create one multi-sample file.
- Copyright (C) 2012-2016 Genome Research Ltd.
+ Copyright (C) 2012-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#include <errno.h>
#include <unistd.h>
#include <getopt.h>
+#include <inttypes.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/vcfutils.h>
typedef struct
{
bcf1_t *line;
- int end, active;
+ int end, active; // end: 0-based INFO/END
}
gvcf_aux_t;
int nfmt_map; // number of rows in the fmt_map array
int *agr_map, nagr_map, magr_map; // mapping between Number=AGR element indexes
void *tmp_arr;
- int ntmp_arr;
+ size_t ntmp_arr;
buffer_t *buf;
AGR_info_t *AGR_info;
int nAGR_info, mAGR_info;
bcf_srs_t *files;
- int gvcf_min, gvcf_break; // min buffered gvcf END position (NB: gvcf_min is 1-based) or 0 if no active lines are present
- gvcf_aux_t *gvcf; // buffer of gVCF lines
+ int gvcf_min, // min buffered gvcf END position (NB: gvcf_min is 1-based) or 0 if no active lines are present
+ gvcf_break; // 0-based position of a next record which breaks a gVCF block
+ gvcf_aux_t *gvcf; // buffer of gVCF lines, for each reader one line
+ int nout_smpl;
+ kstring_t *str;
}
maux_t;
{
int msize = args->maux->ntmp_arr / rule->type_size;
int ret = bcf_get_info_values(hdr, line, rule->hdr_tag, &args->maux->tmp_arr, &msize, rule->type);
- if ( ret<=0 ) error("FIXME: error parsing %s at %s:%d .. %d\n", rule->hdr_tag,bcf_seqname(hdr,line),line->pos+1,ret);
+ if ( ret<=0 ) error("FIXME: error parsing %s at %s:%"PRId64" .. %d\n", rule->hdr_tag,bcf_seqname(hdr,line),(int64_t) line->pos+1,ret);
args->maux->ntmp_arr = msize * rule->type_size;
rule->nblocks++;
int i, j;
if ( var_len==BCF_VL_A )
{
- if ( ret!=line->n_allele-1 ) error("Wrong number of %s fields at %s:%d\n",rule->hdr_tag,bcf_seqname(hdr,line),line->pos+1);
+ if ( ret!=line->n_allele-1 ) error("Wrong number of %s fields at %s:%"PRId64"\n",rule->hdr_tag,bcf_seqname(hdr,line),(int64_t) line->pos+1);
args->maux->nagr_map = ret;
hts_expand(int,args->maux->nagr_map,args->maux->magr_map,args->maux->agr_map);
// create mapping from source file ALT indexes to dst file indexes
}
else if ( var_len==BCF_VL_R )
{
- if ( ret!=line->n_allele ) error("Wrong number of %s fields at %s:%d\n",rule->hdr_tag,bcf_seqname(hdr,line),line->pos+1);
+ if ( ret!=line->n_allele ) error("Wrong number of %s fields at %s:%"PRId64"\n",rule->hdr_tag,bcf_seqname(hdr,line),(int64_t) line->pos+1);
args->maux->nagr_map = ret;
hts_expand(int,args->maux->nagr_map,args->maux->magr_map,args->maux->agr_map);
for (i=0; i<ret; i++) args->maux->agr_map[i] = als->map[i];
else
{
if ( rule->nblocks>1 && ret!=rule->block_size )
- error("Mismatch in number of values for INFO/%s at %s:%d\n", rule->hdr_tag,bcf_seqname(hdr,line),line->pos+1);
+ error("Mismatch in number of values for INFO/%s at %s:%"PRId64"\n", rule->hdr_tag,bcf_seqname(hdr,line),(int64_t) line->pos+1);
rule->block_size = ret;
args->maux->nagr_map = 0;
}
int i;
for (i=0; i<bcf_hdr_nsamples(hr); i++)
{
- char *name = hr->samples[i];
- if ( bcf_hdr_id2int(hw, BCF_DT_SAMPLE, name)!=-1 )
+ char *rmme = NULL, *name = hr->samples[i];
+ while ( bcf_hdr_id2int(hw, BCF_DT_SAMPLE, name)!=-1 )
{
// there is a sample with the same name
if ( !force_samples ) error("Error: Duplicate sample names (%s), use --force-samples to proceed anyway.\n", name);
- int len = strlen(hr->samples[i]) + strlen(clash_prefix) + 1;
- name = (char*) malloc(sizeof(char)*(len+1));
- sprintf(name,"%s:%s",clash_prefix,hr->samples[i]);
- bcf_hdr_add_sample(hw,name);
- free(name);
+ // Resolve conflicting samples names. For example, replace:
+ // A + A with A,2:A
+ // A,2:A + A with A,2:A,2:2:A
+
+ int len = strlen(name) + strlen(clash_prefix) + 1;
+ char *tmp = (char*) malloc(sizeof(char)*(len+1));
+ sprintf(tmp,"%s:%s",clash_prefix,name);
+ free(rmme);
+ rmme = name = tmp;
}
- else
- bcf_hdr_add_sample(hw,name);
+ bcf_hdr_add_sample(hw,name);
+ free(rmme);
}
}
int i, n_smpl = 0;
for (i=0; i<ma->n; i++)
n_smpl += bcf_hdr_nsamples(files->readers[i].header);
+ ma->nout_smpl = n_smpl;
+ assert( n_smpl==bcf_hdr_nsamples(args->out_hdr) );
if ( args->do_gvcf )
{
ma->gvcf = (gvcf_aux_t*) calloc(ma->n,sizeof(gvcf_aux_t));
ma->buf = (buffer_t*) calloc(ma->n,sizeof(buffer_t));
for (i=0; i<ma->n; i++)
ma->buf[i].rid = -1;
+ ma->str = (kstring_t*) calloc(n_smpl,sizeof(kstring_t));
return ma;
}
void maux_destroy(maux_t *ma)
{
int i,j;
+ for (i=0; i<ma->nout_smpl; i++) free(ma->str[i].s);
+ free(ma->str);
for (i=0; i<ma->mals; i++)
{
free(ma->als[i]);
}
ma->buf[i].end = j;
ma->buf[i].cur = -1;
- if ( ma->buf[i].beg < ma->buf[i].end )
+ if ( ma->buf[i].beg < ma->buf[i].end )
{
ma->buf[i].lines = ma->files->readers[i].buffer;
if ( ma->gvcf ) ma->gvcf[i].active = 0; // gvcf block cannot overlap with the next record
int end_src = start_src;
while ( end_src<src_len && src[end_src] && src[end_src]!=',' ) end_src++;
- int nsrc_cpy = end_src - start_src;
+ int nsrc_cpy = end_src - start_src; // number of chars to copy (excluding \0)
if ( nsrc_cpy==1 && src[start_src]=='.' ) return 0; // don't write missing values, dst is already initialized
int ith_dst = 0, start_dst = 0;
agr->mbuf = tmp.m; agr->nbuf = tmp.l; agr->buf = (uint8_t*)tmp.s;
}
else
- error("Not ready for type [%d]: %s at %d\n", info->type,agr->hdr_tag,line->pos+1);
+ error("Not ready for type [%d]: %s at %"PRId64"\n", info->type,agr->hdr_tag,(int64_t) line->pos+1);
}
if ( info->type==BCF_BT_INT8 || info->type==BCF_BT_INT16 || info->type==BCF_BT_INT32 || info->type==BCF_BT_FLOAT )
{
int ret = copy_string_field((char*)info->vptr, iori-ifrom, info->len, &tmp, als->map[iori]-ifrom);
if ( ret )
- error("Error at %s:%d: wrong number of fields in %s?\n", bcf_seqname(hdr,line),line->pos+1,agr->hdr_tag);
+ error("Error at %s:%"PRId64": wrong number of fields in %s?\n", bcf_seqname(hdr,line),(int64_t) line->pos+1,agr->hdr_tag);
}
}
else
int knew = bcf_alleles2gt(inew,jnew);
int ret = copy_string_field((char*)info->vptr, kori, info->len, &tmp, knew);
if ( ret )
- error("Error at %s:%d: wrong number of fields in %s?\n", bcf_seqname(hdr,line),line->pos+1,agr->hdr_tag);
+ error("Error at %s:%"PRId64": wrong number of fields in %s?\n", bcf_seqname(hdr,line),(int64_t) line->pos+1,agr->hdr_tag);
}
}
}
}
kitr = kh_get(strdict, tmph, key);
int idx = kh_val(tmph, kitr);
- if ( idx<0 ) error("Error occurred while processing INFO tag \"%s\" at %s:%d\n", key,bcf_seqname(hdr,line),line->pos+1);
+ if ( idx<0 ) error("Error occurred while processing INFO tag \"%s\" at %s:%"PRId64"\n", key,bcf_seqname(hdr,line),(int64_t) line->pos+1);
merge_AGR_info_tag(hdr, line,inf,len,&ma->buf[i].rec[irec],&ma->AGR_info[idx]);
continue;
}
bcf_hdr_t *out_hdr = args->out_hdr;
maux_t *ma = args->maux;
int i, ismpl = 0, nsamples = bcf_hdr_nsamples(out_hdr);
+ static int warned = 0;
int nsize = 0, msize = sizeof(int32_t);
for (i=0; i<files->nreaders; i++)
{
ma->ntmp_arr = nsamples*nsize*msize;
ma->tmp_arr = realloc(ma->tmp_arr, ma->ntmp_arr);
+ if ( !ma->tmp_arr ) error("Could not allocate %zu bytes\n",ma->ntmp_arr);
+ if ( ma->ntmp_arr > 2147483647 )
+ {
+ if ( !warned ) fprintf(bcftools_stderr,"Warning: Too many genotypes at %s:%"PRId64", requires %zu bytes, skipping.\n", bcf_seqname(out_hdr,out),(int64_t) out->pos+1,ma->ntmp_arr);
+ warned = 1;
+ return;
+ }
}
memset(ma->smpl_ploidy,0,nsamples*sizeof(int));
bcf_update_format_int32(out_hdr, out, "GT", (int32_t*)ma->tmp_arr, nsamples*nsize);
}
+void merge_format_string(args_t *args, const char *key, bcf_fmt_t **fmt_map, bcf1_t *out, int length, int nsize)
+{
+ bcf_srs_t *files = args->files;
+ bcf_hdr_t *out_hdr = args->out_hdr;
+ maux_t *ma = args->maux;
+ int i,j, nsamples = bcf_hdr_nsamples(out_hdr);
+ static int warned = 0;
+
+ // initialize empty strings, a dot for each value, e.g. ".,.,."
+ int nmax = 0;
+ for (i=0; i<nsamples; i++)
+ {
+ kstring_t *str = &ma->str[i];
+ if ( length==BCF_VL_FIXED || length==BCF_VL_VAR )
+ {
+ str->l = 1;
+ ks_resize(str, str->l+1);
+ str->s[0] = '.';
+ }
+ else
+ {
+ str->l = nsize*2 - 1;
+ ks_resize(str, str->l+1);
+ str->s[0] = '.';
+ for (j=1; j<nsize; j++) str->s[j*2-1] = ',', str->s[j*2] = '.';
+ }
+ str->s[str->l] = 0;
+ if ( nmax < str->l ) nmax = str->l;
+ }
+
+ // fill in values for each sample
+ int ismpl = 0;
+ for (i=0; i<files->nreaders; i++)
+ {
+ bcf_sr_t *reader = &files->readers[i];
+ bcf_hdr_t *hdr = reader->header;
+ bcf_fmt_t *fmt_ori = fmt_map[i];
+ if ( !fmt_ori )
+ {
+ // the field is not present in this file
+ ismpl += bcf_hdr_nsamples(hdr);
+ continue;
+ }
+
+ bcf1_t *line = maux_get_line(args, i);
+ int irec = ma->buf[i].cur;
+ char *src = (char*) fmt_ori->p;
+
+ if ( length==BCF_VL_FIXED || length==BCF_VL_VAR || (line->n_allele==out->n_allele && !ma->buf[i].rec[irec].als_differ) )
+ {
+ // alleles unchanged, copy over
+ for (j=0; j<bcf_hdr_nsamples(hdr); j++)
+ {
+ kstring_t *str = &ma->str[ismpl++];
+ str->l = 0;
+ kputsn(src, fmt_ori->n, str);
+ if ( nmax < str->l ) nmax = str->l;
+ src += fmt_ori->n;
+ }
+ continue;
+ }
+ // NB, what is below is not the fastest way, copy_string_field() keeps
+ // finding the indexes repeatedly at multiallelic sites
+ if ( length==BCF_VL_A || length==BCF_VL_R )
+ {
+ int ifrom = length==BCF_VL_A ? 1 : 0;
+ for (j=0; j<bcf_hdr_nsamples(hdr); j++)
+ {
+ kstring_t *str = &ma->str[ismpl++];
+ int iori,inew;
+ for (iori=ifrom; iori<line->n_allele; iori++)
+ {
+ inew = ma->buf[i].rec[irec].map[iori] - ifrom;
+ int ret = copy_string_field(src, iori - ifrom, fmt_ori->size, str, inew);
+ if ( ret<-1 ) error("[E::%s] fixme: internal error at %s:%"PRId64" .. %d\n",__func__,bcf_seqname(hdr,line),(int64_t) line->pos+1,ret);
+ }
+ src += fmt_ori->size;
+ }
+ continue;
+ }
+ assert( length==BCF_VL_G );
+ error("[E::%s] Merging of Number=G FORMAT strings (in your case FORMAT/%s) is not supported yet, sorry!\n"
+ "Please open an issue on github if this feature is essential for you. However, note that using FORMAT strings is not\n"
+ "a good idea in general - it is slow to parse and does not compress well, it is better to use integer codes instead.\n"
+ "If you don't really need it, use `bcftools annotate -x` to remove the annotation before merging.\n", __func__,key);
+ }
+ // update the record
+ if ( ma->ntmp_arr < nsamples*nmax )
+ {
+ ma->ntmp_arr = nsamples*nmax;
+ ma->tmp_arr = realloc(ma->tmp_arr, ma->ntmp_arr);
+ if ( !ma->tmp_arr ) error("Could not allocate %zu bytes\n",ma->ntmp_arr);
+ if ( ma->ntmp_arr > 2147483647 )
+ {
+ if ( !warned ) fprintf(bcftools_stderr,"Warning: The row size is too big for FORMAT/%s at %s:%"PRId64", requires %zu bytes, skipping.\n", key,bcf_seqname(out_hdr,out),(int64_t) out->pos+1,ma->ntmp_arr);
+ warned = 1;
+ return;
+ }
+ }
+ char *tgt = (char*) ma->tmp_arr;
+ for (i=0; i<nsamples; i++)
+ {
+ memcpy(tgt, ma->str[i].s, ma->str[i].l);
+ if ( ma->str[i].l < nmax ) memset(tgt + ma->str[i].l, 0, nmax - ma->str[i].l);
+ tgt += nmax;
+ }
+ bcf_update_format_char(out_hdr, out, key, (float*)ma->tmp_arr, nsamples*nmax);
+}
+
void merge_format_field(args_t *args, bcf_fmt_t **fmt_map, bcf1_t *out)
{
bcf_srs_t *files = args->files;
bcf_hdr_t *out_hdr = args->out_hdr;
maux_t *ma = args->maux;
int i, ismpl = 0, nsamples = bcf_hdr_nsamples(out_hdr);
+ static int warned = 0;
const char *key = NULL;
- int nsize = 0, length = BCF_VL_FIXED, type = -1;
+ size_t nsize = 0, length = BCF_VL_FIXED;
+ int type = -1;
for (i=0; i<files->nreaders; i++)
{
if ( !maux_get_line(args,i) ) continue;
}
if ( fmt_map[i]->n > nsize ) nsize = fmt_map[i]->n;
}
+ if ( type==BCF_BT_CHAR )
+ {
+ merge_format_string(args, key, fmt_map, out, length, nsize);
+ return;
+ }
- int msize = sizeof(float)>sizeof(int32_t) ? sizeof(float) : sizeof(int32_t);
+ size_t msize = sizeof(float)>sizeof(int32_t) ? sizeof(float) : sizeof(int32_t);
if ( ma->ntmp_arr < nsamples*nsize*msize )
{
ma->ntmp_arr = nsamples*nsize*msize;
ma->tmp_arr = realloc(ma->tmp_arr, ma->ntmp_arr);
+ if ( !ma->tmp_arr ) error("Failed to allocate %zu bytes at %s:%"PRId64" for FORMAT/%s\n", ma->ntmp_arr,bcf_seqname(args->out_hdr,out),(int64_t) out->pos+1,key);
+ if ( ma->ntmp_arr > 2147483647 )
+ {
+ if ( !warned ) fprintf(bcftools_stderr,"Warning: The row size is too big for FORMAT/%s at %s:%"PRId64", requires %zu bytes, skipping.\n", key,bcf_seqname(out_hdr,out),(int64_t) out->pos+1,ma->ntmp_arr);
+ warned = 1;
+ return;
+ }
}
// Fill the temp array for all samples by collecting values from all files
bcf_fmt_t *fmt_ori = fmt_map[i];
bcf1_t *line = maux_get_line(args, i);
int irec = ma->buf[i].cur;
+
if ( fmt_ori )
{
type = fmt_ori->type;
{
// if all fields are missing then n==1 is valid
if ( fmt_ori->n!=1 && fmt_ori->n != nals_ori*(nals_ori+1)/2 && fmt_map[i]->n != nals_ori )
- error("Incorrect number of FORMAT/%s values at %s:%d, cannot merge. The tag is defined as Number=G, but found\n"
+ error("Incorrect number of FORMAT/%s values at %s:%"PRId64", cannot merge. The tag is defined as Number=G, but found\n"
"%d values and %d alleles. See also http://samtools.github.io/bcftools/howtos/FAQ.html#incorrect-nfields\n",
- key,bcf_seqname(args->out_hdr,out),out->pos+1,fmt_ori->n,nals_ori);
+ key,bcf_seqname(args->out_hdr,out),(int64_t) out->pos+1,fmt_ori->n,nals_ori);
}
else if ( length==BCF_VL_A )
{
if ( fmt_ori->n!=1 && fmt_ori->n != nals_ori-1 )
- error("Incorrect number of FORMAT/%s values at %s:%d, cannot merge. The tag is defined as Number=A, but found\n"
+ error("Incorrect number of FORMAT/%s values at %s:%"PRId64", cannot merge. The tag is defined as Number=A, but found\n"
"%d values and %d alleles. See also http://samtools.github.io/bcftools/howtos/FAQ.html#incorrect-nfields\n",
- key,bcf_seqname(args->out_hdr,out),out->pos+1,fmt_ori->n,nals_ori);
+ key,bcf_seqname(args->out_hdr,out),(int64_t) out->pos+1,fmt_ori->n,nals_ori);
}
else if ( length==BCF_VL_R )
{
if ( fmt_ori->n!=1 && fmt_ori->n != nals_ori )
- error("Incorrect number of FORMAT/%s values at %s:%d, cannot merge. The tag is defined as Number=R, but found\n"
+ error("Incorrect number of FORMAT/%s values at %s:%"PRId64", cannot merge. The tag is defined as Number=R, but found\n"
"%d values and %d alleles. See also http://samtools.github.io/bcftools/howtos/FAQ.html#incorrect-nfields\n",
- key,bcf_seqname(args->out_hdr,out),out->pos+1,fmt_ori->n,nals_ori);
+ key,bcf_seqname(args->out_hdr,out),(int64_t) out->pos+1,fmt_ori->n,nals_ori);
}
}
case BCF_BT_INT16: BRANCH(int32_t, int16_t, *src==bcf_int16_missing, *src==bcf_int16_vector_end, *tgt=bcf_int32_missing, *tgt=bcf_int32_vector_end); break;
case BCF_BT_INT32: BRANCH(int32_t, int32_t, *src==bcf_int32_missing, *src==bcf_int32_vector_end, *tgt=bcf_int32_missing, *tgt=bcf_int32_vector_end); break;
case BCF_BT_FLOAT: BRANCH(float, float, bcf_float_is_missing(*src), bcf_float_is_vector_end(*src), bcf_float_set_missing(*tgt), bcf_float_set_vector_end(*tgt)); break;
- case BCF_BT_CHAR: BRANCH(uint8_t, uint8_t, *src==bcf_str_missing, *src==bcf_str_vector_end, *tgt=bcf_str_missing, *tgt=bcf_str_vector_end); break;
default: error("Unexpected case: %d, %s\n", type, key);
}
#undef BRANCH
}
if ( type==BCF_BT_FLOAT )
bcf_update_format_float(out_hdr, out, key, (float*)ma->tmp_arr, nsamples*nsize);
- else if ( type==BCF_BT_CHAR )
- bcf_update_format_char(out_hdr, out, key, (float*)ma->tmp_arr, nsamples*nsize);
else
bcf_update_format_int32(out_hdr, out, key, (int32_t*)ma->tmp_arr, nsamples*nsize);
}
{
if ( !gaux[i].active ) continue;
bcf1_t *line = maux_get_line(args, i);
+ if ( !line ) continue;
int irec = maux->buf[i].cur;
hts_expand(int, line->n_allele, maux->buf[i].rec[irec].mmap, maux->buf[i].rec[irec].map);
if ( !maux->als )
{
bcf_hdr_t *hdr = bcf_sr_get_header(args->files,i);
- error("Failed to merge alleles at %s:%d\n",bcf_seqname(hdr,line),line->pos+1);
+ error("Failed to merge alleles at %s:%"PRId64"\n",bcf_seqname(hdr,line),(int64_t) line->pos+1);
}
}
}
/*
Output staged gVCF blocks, end is the last position of the block. Assuming
gaux[i].active flags are set and maux_get_line returns correct lines.
+ Both start,end coordinates are 0-based.
*/
void gvcf_write_block(args_t *args, int start, int end)
{
assert(gaux);
// Update POS
- int min = INT_MAX;
+ int min = INT_MAX; // the minimum active gVCF INFO/END (0-based)
char ref = 'N';
for (i=0; i<args->files->nreaders; i++)
{
if ( min > gaux[i].end ) min = gaux[i].end;
}
// Check for valid gVCF blocks in this region
- if ( min==INT_MAX )
+ if ( min==INT_MAX ) // this probably should not happen
{
assert(0);
maux->gvcf_min = 0;
}
else
bcf_update_info_int32(args->out_hdr, out, "END", NULL, 0);
- bcf_write1(args->out_fh, args->out_hdr, out);
+ if ( bcf_write1(args->out_fh, args->out_hdr, out)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
bcf_clear1(out);
}
// When called on a region, trim the blocks accordingly
- int start = maux->gvcf_break>=0 ? maux->gvcf_break + 1 : maux->pos;
+ int start = maux->gvcf_break>=0 ? maux->gvcf_break + 1 : maux->pos; // the start of a new gvcf block to output
if ( args->regs )
{
int rstart = -1, rend = -1;
// does the block end before the new line or is it interrupted?
int tmp = maux->gvcf_min < flush_until ? maux->gvcf_min : flush_until;
if ( start > tmp-1 ) break;
- gvcf_write_block(args,start,tmp-1); // gvcf_min is 1-based
+ gvcf_write_block(args,start,tmp-1); // gvcf_min is 1-based, passing 0-based coordinates
start = tmp;
}
}
Check incoming lines for new gVCF blocks, set pointer to the current source
buffer (gvcf or readers). In contrast to gvcf_flush, this function can be
called only after maux_reset as it relies on updated maux buffers.
+ The coordinate is 0-based
*/
void gvcf_stage(args_t *args, int pos)
{
int ret = bcf_get_info_int32(hdr,line,"END",&end,&nend);
if ( ret==1 )
{
+ if ( end[0] == line->pos + 1 ) // POS and INFO/END are identical, treat as if a normal w/o INFO/END
+ {
+ maux->gvcf_break = line->pos;
+ continue;
+ }
+ if ( end[0] <= line->pos ) error("Error: Incorrect END at %s:%"PRId64" .. END=%d\n", bcf_seqname(hdr,line),(int64_t) line->pos+1,end[0]);
+
// END is set, this is a new gVCF block. Cache this line in gaux[i] and swap with
// an empty record: the gaux line must be kept until we reach its END.
+
gaux[i].active = 1;
gaux[i].end = end[0] - 1;
SWAP(bcf1_t*,args->files->readers[i].buffer[irec],gaux[i].line);
{
// Invalidate pointer to reader's buffer or else gvcf_flush will attempt
// to use the old lines via maux_get_line()
- if ( ma->gvcf && !ma->gvcf[ir].active ) ma->buf[ir].cur = -1;
+ if ( ma->gvcf )
+ {
+ if ( ma->gvcf[ir].active )
+ {
+ if ( ma->pos >= ma->gvcf[ir].end ) ma->gvcf[ir].active = 0;
+ else if ( ma->buf[ir].cur==-1 ) ma->buf[ir].cur = ma->buf[ir].beg; // re-activate interrupted gVCF block
+ }
+ if ( !ma->gvcf[ir].active ) ma->buf[ir].cur = -1;
+ }
bcf_sr_t *reader = bcf_sr_get_reader(args->files,ir);
if ( !reader->nbuffer ) continue; // nothing to clean
bcf_hdr_t *hdr = bcf_sr_get_header(args->files,i);
const char *chr = bcf_hdr_id2name(hdr, maux->buf[i].rid);
fprintf(bcftools_stderr,"\t");
- for (j=maux->buf[i].beg; j<maux->buf[i].end; j++) fprintf(bcftools_stderr," %s:%d",chr,maux->buf[i].lines[j]->pos+1);
+ for (j=maux->buf[i].beg; j<maux->buf[i].end; j++) fprintf(bcftools_stderr," %s:%"PRId64,chr,(int64_t) maux->buf[i].lines[j]->pos+1);
}
fprintf(bcftools_stderr,"\n");
}
+ fprintf(bcftools_stderr,"gvcf_min=%d\n", args->maux->gvcf_min);
for (i=0; i<args->files->nreaders; i++)
{
fprintf(bcftools_stderr,"reader %d:\tgvcf_active=%d", i,maux->gvcf[i].active);
- if ( maux->gvcf[i].active ) fprintf(bcftools_stderr,"\tpos,end=%d,%d", maux->gvcf[i].line->pos+1,maux->gvcf[i].end+1);
+ if ( maux->gvcf[i].active ) fprintf(bcftools_stderr,"\tpos,end=%"PRId64",%"PRId64, (int64_t) maux->gvcf[i].line->pos+1,(int64_t) maux->gvcf[i].end+1);
fprintf(bcftools_stderr,"\n");
}
fprintf(bcftools_stderr,"\n");
}
// normalize alleles
maux->als = merge_alleles(line->d.allele, line->n_allele, buf->rec[j].map, maux->als, &maux->nals, &maux->mals);
- if ( !maux->als ) error("Failed to merge alleles at %s:%d in %s\n",maux->chr,line->pos+1,reader->fname);
+ if ( !maux->als ) error("Failed to merge alleles at %s:%"PRId64" in %s\n",maux->chr,(int64_t) line->pos+1,reader->fname);
hts_expand0(int, maux->nals, maux->ncnt, maux->cnt);
for (k=1; k<line->n_allele; k++)
maux->cnt[ buf->rec[j].map[k] ]++; // how many times an allele appears in the files
if ( args->do_gvcf )
bcf_update_info_int32(args->out_hdr, out, "END", NULL, 0);
merge_format(args, out);
- bcf_write1(args->out_fh, args->out_hdr, out);
+ if ( bcf_write1(args->out_fh, args->out_hdr, out)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
bcf_clear1(out);
}
void bcf_hdr_append_version(bcf_hdr_t *hdr, int argc, char **argv, const char *cmd)
{
kstring_t str = {0,0,0};
- ksprintf(&str,"##%sVersion=%s+htslib-%s\n", cmd, bcftools_version(), hts_version());
- bcf_hdr_append(hdr,str.s);
+ int e = 0;
+ if (ksprintf(&str,"##%sVersion=%s+htslib-%s\n", cmd, bcftools_version(), hts_version()) < 0)
+ goto fail;
+ if (bcf_hdr_append(hdr,str.s) < 0)
+ goto fail;
str.l = 0;
- ksprintf(&str,"##%sCommand=%s", cmd, argv[0]);
+ e |= ksprintf(&str,"##%sCommand=%s", cmd, argv[0]) < 0;
int i;
for (i=1; i<argc; i++)
{
if ( strchr(argv[i],' ') )
- ksprintf(&str, " '%s'", argv[i]);
+ e |= ksprintf(&str, " '%s'", argv[i]) < 0;
else
- ksprintf(&str, " %s", argv[i]);
- }
- kputs("; Date=", &str);
- time_t tm; time(&tm); kputs(ctime(&tm), &str);
- kputc('\n', &str);
- bcf_hdr_append(hdr,str.s);
+ e |= ksprintf(&str, " %s", argv[i]) < 0;
+ }
+ e |= kputs("; Date=", &str) < 0;
+ time_t tm; time(&tm);
+ e |= kputs(ctime(&tm), &str) < 0;
+ e |= kputc('\n', &str) < 0;
+ if (e)
+ goto fail;
+ if (bcf_hdr_append(hdr,str.s) < 0)
+ goto fail;
+ free(ks_release(&str));
+
+ if (bcf_hdr_sync(hdr) < 0)
+ goto fail;
+ return;
+
+ fail:
free(str.s);
-
- bcf_hdr_sync(hdr);
+ error_errno("[%s] Failed to add program information to header", __func__);
}
void merge_vcf(args_t *args)
int i;
for (i=0; i<args->files->nreaders; i++)
{
- char buf[10]; snprintf(buf,10,"%d",i+1);
+ char buf[24]; snprintf(buf,sizeof buf,"%d",i+1);
merge_headers(args->out_hdr, args->files->readers[i].header,buf,args->force_samples);
}
if (args->record_cmd_line) bcf_hdr_append_version(args->out_hdr, args->argc, args->argv, "bcftools_merge");
- bcf_hdr_sync(args->out_hdr);
+ if (bcf_hdr_sync(args->out_hdr) < 0)
+ error_errno("[%s] Failed to update header", __func__);
}
info_rules_init(args);
bcf_hdr_set_version(args->out_hdr, bcf_hdr_get_version(args->files->readers[0].header));
- bcf_hdr_write(args->out_fh, args->out_hdr);
+ if ( bcf_hdr_write(args->out_fh, args->out_hdr)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
if ( args->header_only )
{
bcf_hdr_destroy(args->out_hdr);
- hts_close(args->out_fh);
+ if ( hts_close(args->out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
return;
}
info_rules_destroy(args);
maux_destroy(args->maux);
bcf_hdr_destroy(args->out_hdr);
- hts_close(args->out_fh);
+ if ( hts_close(args->out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
bcf_destroy1(args->out_line);
kh_destroy(strdict, args->tmph);
if ( args->tmps.m ) free(args->tmps.s);
fprintf(bcftools_stderr, " -O, --output-type <b|u|z|v> 'b' compressed BCF; 'u' uncompressed BCF; 'z' compressed VCF; 'v' uncompressed VCF [v]\n");
fprintf(bcftools_stderr, " -r, --regions <region> restrict to comma-separated list of regions\n");
fprintf(bcftools_stderr, " -R, --regions-file <file> restrict to regions listed in a file\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(bcftools_stderr, "\n");
exit(1);
}
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
/* vcfnorm.c -- Left-align and normalize indels.
- Copyright (C) 2013-2017 Genome Research Ltd.
+ Copyright (C) 2013-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
+#include <inttypes.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/faidx.h>
#include "bcftools.h"
#include "rbuf.h"
-#define CHECK_REF_EXIT 0
-#define CHECK_REF_WARN 1
-#define CHECK_REF_SKIP 2
-#define CHECK_REF_FIX 4
+#define CHECK_REF_EXIT 1
+#define CHECK_REF_WARN 2
+#define CHECK_REF_SKIP 4
+#define CHECK_REF_FIX 8
#define MROWS_SPLIT 1
#define MROWS_MERGE 2
char *ref, *alt;
void *hash;
}
+cmpals1_t;
+
+typedef struct
+{
+ cmpals1_t *cmpals;
+ int ncmpals, mcmpals;
+}
cmpals_t;
typedef struct
int aln_win; // the realignment window size (maximum repeat size)
bcf_srs_t *files; // using the synced reader only for -r option
bcf_hdr_t *hdr;
- cmpals_t *cmpals;
- int ncmpals, mcmpals;
+ cmpals_t cmpals_in, cmpals_out;
faidx_t *fai;
struct { int tot, set, swap; } nref;
char **argv, *output_fname, *ref_fname, *vcf_fname, *region, *targets;
int argc, rmdup, output_type, n_threads, check_ref, strict_filter, do_indels;
int nchanged, nskipped, nsplit, ntotal, mrows_op, mrows_collapse, parsimonious;
- int record_cmd_line;
+ int record_cmd_line, force, force_warned;
}
args_t;
}
char *ref = faidx_fetch_seq(args->fai, (char*)bcf_seqname(args->hdr,line), line->pos, line->pos+maxlen-1, &len);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !ref ) error("faidx_fetch_seq failed at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
replace_iupac_codes(ref,len);
args->nref.tot++;
int i, j, nals = line->n_allele, nals_ori = line->n_allele;
for (i=1, j=1; i<line->n_allele; i++)
{
- if ( strcmp(line->d.allele[0],line->d.allele[i]) )
+ if ( strcasecmp(line->d.allele[0],line->d.allele[i]) )
{
args->tmp_arr1[i] = j++;
continue;
// Sanity check REF
int i, nref, reflen = strlen(line->d.allele[0]);
char *ref = faidx_fetch_seq(args->fai, (char*)args->hdr->id[BCF_DT_CTG][line->rid].key, line->pos, line->pos+reflen-1, &nref);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", args->hdr->id[BCF_DT_CTG][line->rid].key, line->pos+1);
+ if ( !ref ) error("faidx_fetch_seq failed at %s:%"PRId64"\n", args->hdr->id[BCF_DT_CTG][line->rid].key, (int64_t) line->pos+1);
seq_to_upper(ref,0);
replace_iupac_codes(ref,nref); // any non-ACGT character in fasta ref is replaced with N
if ( has_non_acgtn(line->d.allele[0],reflen) )
{
if ( args->check_ref==CHECK_REF_EXIT )
- error("Non-ACGTN reference allele at %s:%d .. REF_SEQ:'%s' vs VCF:'%s'\n", bcf_seqname(args->hdr,line),line->pos+1,ref,line->d.allele[0]);
+ error("Non-ACGTN reference allele at %s:%"PRId64" .. REF_SEQ:'%s' vs VCF:'%s'\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,ref,line->d.allele[0]);
if ( args->check_ref & CHECK_REF_WARN )
- fprintf(stderr,"NON_ACGTN_REF\t%s\t%d\t%s\n", bcf_seqname(args->hdr,line),line->pos+1,line->d.allele[0]);
+ fprintf(stderr,"NON_ACGTN_REF\t%s\t%"PRId64"\t%s\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,line->d.allele[0]);
free(ref);
return ERR_REF_MISMATCH;
}
if ( strcasecmp(ref,line->d.allele[0]) )
{
if ( args->check_ref==CHECK_REF_EXIT )
- error("Reference allele mismatch at %s:%d .. REF_SEQ:'%s' vs VCF:'%s'\n", bcf_seqname(args->hdr,line),line->pos+1,ref,line->d.allele[0]);
+ error("Reference allele mismatch at %s:%"PRId64" .. REF_SEQ:'%s' vs VCF:'%s'\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,ref,line->d.allele[0]);
if ( args->check_ref & CHECK_REF_WARN )
- fprintf(stderr,"REF_MISMATCH\t%s\t%d\t%s\n", bcf_seqname(args->hdr,line),line->pos+1,line->d.allele[0]);
+ fprintf(stderr,"REF_MISMATCH\t%s\t%"PRId64"\t%s\t%s\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,line->d.allele[0],ref);
free(ref);
return ERR_REF_MISMATCH;
}
if ( has_non_acgtn(line->d.allele[i],line->shared.l) )
{
if ( args->check_ref==CHECK_REF_EXIT )
- error("Non-ACGTN alternate allele at %s:%d .. REF_SEQ:'%s' vs VCF:'%s'\n", bcf_seqname(args->hdr,line),line->pos+1,ref,line->d.allele[i]);
+ error("Non-ACGTN alternate allele at %s:%"PRId64" .. VCF:'%s'\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,line->d.allele[i]);
if ( args->check_ref & CHECK_REF_WARN )
- fprintf(stderr,"NON_ACGTN_ALT\t%s\t%d\t%s\n", bcf_seqname(args->hdr,line),line->pos+1,line->d.allele[i]);
+ fprintf(stderr,"NON_ACGTN_ALT\t%s\t%"PRId64"\t%s\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,line->d.allele[i]);
return ERR_REF_MISMATCH;
}
kputs(line->d.allele[i], &als[i]);
seq_to_upper(als[i].s,0);
- if ( i>0 && als[i].l==als[0].l && !strcmp(als[0].s,als[i].s) ) return ERR_DUP_ALLELE;
+ if ( i>0 && als[i].l==als[0].l && !strcasecmp(als[0].s,als[i].s) ) return ERR_DUP_ALLELE;
}
// trim from right
int min_len = als[0].l;
for (i=1; i<line->n_allele; i++)
{
- if ( als[0].s[ als[0].l-1 ]!=als[i].s[ als[i].l-1 ] ) break;
+ if ( toupper(als[0].s[ als[0].l-1 ])!=toupper(als[i].s[ als[i].l-1 ]) ) break;
if ( als[i].l < min_len ) min_len = als[i].l;
}
if ( i!=line->n_allele ) break; // there are differences, cannot be trimmed
int npad = line->pos >= args->aln_win ? args->aln_win : line->pos;
free(ref);
ref = faidx_fetch_seq(args->fai, (char*)args->hdr->id[BCF_DT_CTG][line->rid].key, line->pos-npad, line->pos-1, &nref);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", args->hdr->id[BCF_DT_CTG][line->rid].key, line->pos-npad+1);
+ if ( !ref ) error("faidx_fetch_seq failed at %s:%"PRId64"\n", args->hdr->id[BCF_DT_CTG][line->rid].key, (int64_t) line->pos-npad+1);
replace_iupac_codes(ref,nref);
for (i=0; i<line->n_allele; i++)
{
// Have the alleles changed?
als[0].s[ als[0].l ] = 0; // in order for strcmp to work
- if ( ori_pos==line->pos && !strcmp(line->d.allele[0],als[0].s) ) return ERR_OK;
+ if ( ori_pos==line->pos && !strcasecmp(line->d.allele[0],als[0].s) ) return ERR_OK;
// Create new block of alleles and update
args->tmp_als_str.l = 0;
if ( len==BCF_VL_A ) \
{ \
if ( ret!=src->n_allele-1 ) \
- error("Error: wrong number of fields in INFO/%s at %s:%d, expected %d, found %d\n", \
- tag,bcf_seqname(args->hdr,src),src->pos+1,src->n_allele-1,ret); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(stderr, \
+ "Warning: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele-1,ret); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_info_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele-1,ret); \
+ } \
bcf_update_info_##type(args->hdr,dst,tag,vals+ialt,1); \
} \
else if ( len==BCF_VL_R ) \
{ \
if ( ret!=src->n_allele ) \
- error("Error: wrong number of fields in INFO/%s at %s:%d, expected %d, found %d\n", \
- tag,bcf_seqname(args->hdr,src),src->pos+1,src->n_allele,ret); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(stderr, \
+ "Warning: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele,ret); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_info_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele,ret); \
+ } \
if ( ialt!=0 ) vals[1] = vals[ialt+1]; \
bcf_update_info_##type(args->hdr,dst,tag,vals,2); \
} \
else if ( len==BCF_VL_G ) \
{ \
if ( ret!=src->n_allele*(src->n_allele+1)/2 ) \
- error("Error: wrong number of fields in INFO/%s at %s:%d, expected %d, found %d\n", \
- tag,bcf_seqname(args->hdr,src),src->pos+1,src->n_allele*(src->n_allele+1)/2,ret); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(stderr, \
+ "Warning: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele*(src->n_allele+1)/2,ret); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_info_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele*(src->n_allele+1)/2,ret); \
+ } \
if ( ialt!=0 ) \
{ \
vals[1] = vals[bcf_alleles2gt(0,ialt+1)]; \
if ( len==BCF_VL_A ) \
{ \
if ( nvals!=(src->n_allele-1)*nsmpl ) \
- error("Error: wrong number of fields in FMT/%s at %s:%d, expected %d, found %d\n", \
- tag,bcf_seqname(args->hdr,src),src->pos+1,(src->n_allele-1)*nsmpl,nvals); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(stderr, \
+ "Warning: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d, found %d. Removing the field.\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,(src->n_allele-1)*nsmpl,nvals); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_format_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d, found %d\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,(src->n_allele-1)*nsmpl,nvals); \
+ } \
nvals /= nsmpl; \
type_t *src_vals = vals, *dst_vals = vals; \
for (i=0; i<nsmpl; i++) \
else if ( len==BCF_VL_R ) \
{ \
if ( nvals!=src->n_allele*nsmpl ) \
- error("Error: wrong number of fields in FMT/%s at %s:%d, expected %d, found %d\n", \
- tag,bcf_seqname(args->hdr,src),src->pos+1,src->n_allele*nsmpl,nvals); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(stderr, \
+ "Warning: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d, found %d. Removing the field.\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,(src->n_allele-1)*nsmpl,nvals); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_format_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d, found %d\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele*nsmpl,nvals); \
+ } \
nvals /= nsmpl; \
type_t *src_vals = vals, *dst_vals = vals; \
for (i=0; i<nsmpl; i++) \
else if ( len==BCF_VL_G ) \
{ \
if ( nvals!=src->n_allele*(src->n_allele+1)/2*nsmpl && nvals!=src->n_allele*nsmpl ) \
- error("Error at %s:%d, the tag %s has wrong number of fields\n", bcf_seqname(args->hdr,src),src->pos+1,bcf_hdr_int2id(args->hdr,BCF_DT_ID,fmt->id)); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(stderr, \
+ "Warning: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d, found %d. Removing the field.\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,(src->n_allele-1)*nsmpl,nvals); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_format_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error at %s:%"PRId64", the tag %s has wrong number of fields\n", bcf_seqname(args->hdr,src),(int64_t) src->pos+1,bcf_hdr_int2id(args->hdr,BCF_DT_ID,fmt->id)); \
+ } \
nvals /= nsmpl; \
int all_haploid = nvals==src->n_allele ? 1 : 0; \
type_t *src_vals = vals, *dst_vals = vals; \
{
const char *tag = bcf_hdr_int2id(args->hdr,BCF_DT_ID,fmt->id);
int ret = bcf_get_format_char(args->hdr,src,tag,&args->tmp_arr1,&args->ntmp_arr1);
+ if ( !ret ) return; // all values can be empty, leave out the tag, no need to panic
assert( ret>0 );
kstring_t str;
if ( *se==',' ) nfields++;
se++;
}
+ if ( nfields==1 && se-ptr==1 && *ptr=='.' ) continue; // missing value
if ( nfields!=src->n_allele*(src->n_allele+1)/2 && nfields!=src->n_allele )
- error("Error: wrong number of fields in FMT/%s at %s:%d, expected %d or %d, found %d\n",
- tag,bcf_seqname(args->hdr,src),src->pos+1,src->n_allele*(src->n_allele+1)/2,src->n_allele,nfields);
+ {
+ if ( args->force && !args->force_warned )
+ {
+ fprintf(stderr,
+ "Warning: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d or %d, found %d. Removing the field.\n"
+ " (This warning is printed only once.)\n",
+ tag,bcf_seqname(args->hdr,src),(int64_t)src->pos+1,src->n_allele*(src->n_allele+1)/2,src->n_allele,nfields);
+ args->force_warned = 1;
+ }
+ if ( args->force )
+ {
+ bcf_update_format_char(args->hdr,dst,tag,NULL,0);
+ return;
+ }
+ error("Error: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d or %d, found %d\n",
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele*(src->n_allele+1)/2,src->n_allele,nfields);
+ }
int len = 0;
if ( nfields==src->n_allele ) // haploid
if ( len==BCF_VL_A ) \
{ \
if (nvals_ori!=lines[0]->n_allele - 1) \
- error("vcfnorm: number of fields in first record at position %s:%d for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele-1); \
+ error("vcfnorm: number of fields in first record at position %s:%"PRId64" for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),(int64_t) lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele-1); \
int nvals = dst->n_allele - 1; \
ENLARGE_ARRAY(type_t,set_missing,args->tmp_arr1,args->ntmp_arr1,1,nvals_ori,nvals); \
vals = (type_t*) args->tmp_arr1; \
if (nvals2<0) continue; /* info tag does not exist in this record, skip */ \
args->ntmp_arr2 = ntmp2 * sizeof(type_t); \
if (nvals2!=lines[i]->n_allele-1) \
- error("vcfnorm: could not merge INFO tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge INFO tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals2 = (type_t*) args->tmp_arr2; \
for (k=0; k<nvals2; k++) \
{ \
else if ( len==BCF_VL_R ) \
{ \
if (nvals_ori!=lines[0]->n_allele) \
- error("vcfnorm: number of fields in first record at position %s:%d for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele); \
+ error("vcfnorm: number of fields in first record at position %s:%"PRId64" for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),(int64_t) lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele); \
int nvals = dst->n_allele; \
ENLARGE_ARRAY(type_t,set_missing,args->tmp_arr1,args->ntmp_arr1,1,nvals_ori,nvals); \
vals = (type_t*) args->tmp_arr1; \
if (nvals2<0) continue; /* info tag does not exist in this record, skip */ \
args->ntmp_arr2 = ntmp2 * sizeof(type_t); \
if (nvals2!=lines[i]->n_allele) \
- error("vcfnorm: could not merge INFO tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge INFO tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals2 = (type_t*) args->tmp_arr2; \
for (k=0; k<nvals2; k++) \
{ \
/* expecting diploid gt in INFO */ \
if (nvals_ori!=lines[0]->n_allele*(lines[0]->n_allele+1)/2) { \
fprintf(stderr, "todo: merge Number=G INFO fields for haploid sites\n"); \
- error("vcfnorm: number of fields in first record at position %s:%d for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele*(lines[0]->n_allele+1)/2); \
+ error("vcfnorm: number of fields in first record at position %s:%"PRId64" for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),(int64_t) lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele*(lines[0]->n_allele+1)/2); \
} \
int nvals = dst->n_allele*(dst->n_allele+1)/2; \
ENLARGE_ARRAY(type_t,set_missing,args->tmp_arr1,args->ntmp_arr1,1,nvals_ori,nvals); \
if (nvals2<0) continue; /* info tag does not exist in this record, skip */ \
args->ntmp_arr2 = ntmp2 * sizeof(type_t); \
if (nvals2!=lines[i]->n_allele*(lines[i]->n_allele+1)/2) \
- error("vcfnorm: could not merge INFO tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge INFO tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals2 = (type_t*) args->tmp_arr2; \
int ia,ib; \
k = 0; \
int ngts2 = bcf_get_genotypes(args->hdr,lines[i],&args->tmp_arr2,&ntmp2);
args->ntmp_arr2 = ntmp2 * 4;
ngts2 /= nsmpl;
- if ( ngts!=ngts2 ) error("Error at %s:%d: cannot combine diploid with haploid genotype\n", bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1);
+ if ( ngts!=ngts2 ) error("Error at %s:%"PRId64": cannot combine diploid with haploid genotype\n", bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1);
int32_t *gt = (int32_t*) args->tmp_arr1;
int32_t *gt2 = (int32_t*) args->tmp_arr2;
else
{
int ial = bcf_gt_allele(gt2[k]);
- if ( ial>=args->maps[i].nals ) error("Error at %s:%d: incorrect allele index %d\n",bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1,ial);
+ if ( ial>=args->maps[i].nals ) error("Error at %s:%"PRId64": incorrect allele index %d\n",bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1,ial);
gt[k] = bcf_gt_unphased( args->maps[i].map[ial] ) | bcf_gt_is_phased(gt[k]);
}
}
args->ntmp_arr2 = ntmp2 * sizeof(type_t); \
nvals2 /= nsmpl; \
if (nvals2!=lines[i]->n_allele-1) \
- error("vcfnorm: could not merge FORMAT tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge FORMAT tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals = (type_t*) args->tmp_arr1; \
vals2 = (type_t*) args->tmp_arr2; \
for (j=0; j<nsmpl; j++) \
args->ntmp_arr2 = ntmp2 * sizeof(type_t); \
nvals2 /= nsmpl; \
if (nvals2!=lines[i]->n_allele) \
- error("vcfnorm: could not merge FORMAT tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge FORMAT tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals = (type_t*) args->tmp_arr1; \
vals2 = (type_t*) args->tmp_arr2; \
for (j=0; j<nsmpl; j++) \
int ndiploid = lines[i]->n_allele*(lines[i]->n_allele+1)/2; \
int line_diploid = nvals2==ndiploid ? 1 : 0; \
if (!(nvals2==1 || nvals2==lines[i]->n_allele || nvals2==lines[i]->n_allele*(lines[i]->n_allele+1)/2)) \
- error("vcfnorm: could not merge FORMAT tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge FORMAT tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals = (type_t*) args->tmp_arr1; \
vals2 = (type_t*) args->tmp_arr2; \
for (j=0; j<nsmpl; j++) \
haploid[i] = 0;
nfields = dst->n_allele*(dst->n_allele+1)/2;
}
- else error("The field %s at %s:%d neither diploid nor haploid?\n", tag,bcf_seqname(args->hdr,dst),dst->pos+1);
+ else error("The field %s at %s:%"PRId64" neither diploid nor haploid?\n", tag,bcf_seqname(args->hdr,dst),(int64_t) dst->pos+1);
kstring_t *tmp = &args->tmp_str[i];
kputc('.',tmp);
args->maps[i].nals = lines[i]->n_allele;
hts_expand(int,args->maps[i].nals,args->maps[i].mals,args->maps[i].map);
args->als = merge_alleles(lines[i]->d.allele, lines[i]->n_allele, args->maps[i].map, args->als, &args->nals, &args->mals);
- if ( !args->als ) error("Failed to merge alleles at %s:%d\n", bcf_seqname(args->hdr,dst),dst->pos+1);
+ if ( !args->als ) error("Failed to merge alleles at %s:%"PRId64"\n", bcf_seqname(args->hdr,dst),(int64_t) dst->pos+1);
}
bcf_update_alleles(args->hdr, dst, (const char**)args->als, args->nals);
for (i=0; i<args->nals; i++)
}
return NULL;
}
-static void cmpals_add(args_t *args, bcf1_t *rec)
+static void cmpals_add(cmpals_t *ca, bcf1_t *rec)
{
- args->ncmpals++;
- hts_expand0(cmpals_t, args->ncmpals, args->mcmpals, args->cmpals);
- cmpals_t *cmpals = args->cmpals + args->ncmpals - 1;
+ ca->ncmpals++;
+ hts_expand0(cmpals1_t, ca->ncmpals, ca->mcmpals, ca->cmpals);
+ cmpals1_t *cmpals = ca->cmpals + ca->ncmpals - 1;
free(cmpals->ref);
cmpals->ref = strdup(rec->d.allele[0]);
cmpals->n = rec->n_allele;
khash_str2int_inc(cmpals->hash, strdup(rec->d.allele[i]));
}
}
-static int cmpals_match(args_t *args, bcf1_t *rec)
+static int cmpals_match(cmpals_t *ca, bcf1_t *rec)
{
int i, j;
- for (i=0; i<args->ncmpals; i++)
+ for (i=0; i<ca->ncmpals; i++)
{
- cmpals_t *cmpals = args->cmpals + i;
+ cmpals1_t *cmpals = ca->cmpals + i;
if ( rec->n_allele != cmpals->n ) continue;
// NB. assuming both are normalized
- if ( strcmp(rec->d.allele[0], cmpals->ref) ) continue;
+ if ( strcasecmp(rec->d.allele[0], cmpals->ref) ) continue;
// the most frequent case
if ( rec->n_allele==2 )
{
- if ( strcmp(rec->d.allele[1], cmpals->alt) ) continue;
+ if ( strcasecmp(rec->d.allele[1], cmpals->alt) ) continue;
return 1;
}
if ( j<rec->n_allele ) continue;
return 1;
}
- cmpals_add(args, rec);
return 0;
}
-static void cmpals_reset(args_t *args) { args->ncmpals = 0; }
-static void cmpals_destroy(args_t *args)
+static void cmpals_reset(cmpals_t *ca) { ca->ncmpals = 0; }
+static void cmpals_destroy(cmpals_t *ca)
{
int i;
- for (i=0; i<args->mcmpals; i++)
+ for (i=0; i<ca->mcmpals; i++)
{
- cmpals_t *cmpals = args->cmpals + i;
+ cmpals1_t *cmpals = ca->cmpals + i;
free(cmpals->ref);
free(cmpals->alt);
if ( cmpals->hash ) khash_str2int_destroy_free(cmpals->hash);
}
- free(args->cmpals);
+ free(ca->cmpals);
}
static void flush_buffer(args_t *args, htsFile *file, int n)
{
if ( mrows_ready_to_flush(args, args->lines[k]) )
{
- while ( (line=mrows_flush(args)) ) bcf_write1(file, args->hdr, line);
+ while ( (line=mrows_flush(args)) )
+ if ( bcf_write1(file, args->hdr, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
int merge = 1;
if ( args->mrows_collapse!=COLLAPSE_BOTH && args->mrows_collapse!=COLLAPSE_ANY )
if ( args->rmdup & BCF_SR_PAIR_ANY ) continue; // rmdup by position only
if ( args->rmdup & BCF_SR_PAIR_SNPS && line_type&(VCF_SNP|VCF_MNP) && prev_type&(VCF_SNP|VCF_MNP) ) continue;
if ( args->rmdup & BCF_SR_PAIR_INDELS && line_type&(VCF_INDEL) && prev_type&(VCF_INDEL) ) continue;
- if ( args->rmdup & BCF_SR_PAIR_EXACT && cmpals_match(args, args->lines[k]) ) continue;
+ if ( args->rmdup & BCF_SR_PAIR_EXACT && cmpals_match(&args->cmpals_out, args->lines[k]) ) continue;
}
else
{
prev_rid = args->lines[k]->rid;
prev_pos = args->lines[k]->pos;
prev_type = 0;
- if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_reset(args);
+ if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_reset(&args->cmpals_out);
}
prev_type |= line_type;
- if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_add(args, args->lines[k]);
+ if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_add(&args->cmpals_out, args->lines[k]);
}
- bcf_write1(file, args->hdr, args->lines[k]);
+ if ( bcf_write1(file, args->hdr, args->lines[k])!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
if ( args->mrows_op==MROWS_MERGE && !args->rbuf.n )
{
- while ( (line=mrows_flush(args)) ) bcf_write1(file, args->hdr, line);
+ while ( (line=mrows_flush(args)) )
+ if ( bcf_write1(file, args->hdr, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
}
static void destroy_data(args_t *args)
{
- cmpals_destroy(args);
+ cmpals_destroy(&args->cmpals_in);
+ cmpals_destroy(&args->cmpals_out);
int i;
for (i=0; i<args->rbuf.m; i++)
if ( args->lines[i] ) bcf_destroy1(args->lines[i]);
if ( args->check_ref & CHECK_REF_FIX )
fix_dup_alt(args, line);
else if ( args->check_ref==CHECK_REF_EXIT )
- error("Duplicate alleles at %s:%d; run with -cw to turn the error into warning or with -cs to fix.\n", bcf_seqname(args->hdr,line),line->pos+1);
+ error("Duplicate alleles at %s:%"PRId64"; run with -cw to turn the error into warning or with -cs to fix.\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
else if ( args->check_ref & CHECK_REF_WARN )
- fprintf(stderr,"ALT_DUP\t%s\t%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ fprintf(stderr,"ALT_DUP\t%s\t%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
}
}
}
if ( args->n_threads )
hts_set_opt(out, HTS_OPT_THREAD_POOL, args->files->p);
if (args->record_cmd_line) bcf_hdr_append_version(args->hdr, args->argc, args->argv, "bcftools_norm");
- bcf_hdr_write(out, args->hdr);
+ if ( bcf_hdr_write(out, args->hdr)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
int prev_rid = -1, prev_pos = -1, prev_type = 0;
while ( bcf_sr_next_line(args->files) )
if ( args->rmdup & BCF_SR_PAIR_ANY ) continue; // rmdup by position only
if ( args->rmdup & BCF_SR_PAIR_SNPS && line_type&(VCF_SNP|VCF_MNP) && prev_type&(VCF_SNP|VCF_MNP) ) continue;
if ( args->rmdup & BCF_SR_PAIR_INDELS && line_type&(VCF_INDEL) && prev_type&(VCF_INDEL) ) continue;
- if ( args->rmdup & BCF_SR_PAIR_EXACT && cmpals_match(args, line) ) continue;
+ if ( args->rmdup & BCF_SR_PAIR_EXACT && cmpals_match(&args->cmpals_in, line) ) continue;
}
else
{
prev_rid = line->rid;
prev_pos = line->pos;
prev_type = 0;
- if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_reset(args);
+ if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_reset(&args->cmpals_in);
}
prev_type |= line_type;
- if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_add(args, line);
+ if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_add(&args->cmpals_in, line);
}
// still on the same chromosome?
if ( j>0 ) flush_buffer(args, out, j);
}
flush_buffer(args, out, args->rbuf.n);
- hts_close(out);
+ if ( hts_close(out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
fprintf(stderr,"Lines total/split/realigned/skipped:\t%d/%d/%d/%d\n", args->ntotal,args->nsplit,args->nchanged,args->nskipped);
if ( args->check_ref & CHECK_REF_FIX )
fprintf(stderr, "Options:\n");
fprintf(stderr, " -c, --check-ref <e|w|x|s> check REF alleles and exit (e), warn (w), exclude (x), or set (s) bad sites [e]\n");
fprintf(stderr, " -D, --remove-duplicates remove duplicate lines of the same type.\n");
- fprintf(stderr, " -d, --rm-dup <type> remove duplicate snps|indels|both|all|none\n");
- fprintf(stderr, " -f, --fasta-ref <file> reference sequence (MANDATORY)\n");
+ fprintf(stderr, " -d, --rm-dup <type> remove duplicate snps|indels|both|all|exact\n");
+ fprintf(stderr, " -f, --fasta-ref <file> reference sequence\n");
+ fprintf(stderr, " --force try to proceed even if malformed tags are encountered. Experimental, use at your own risk\n");
fprintf(stderr, " -m, --multiallelics <-|+>[type] split multiallelics (-) or join biallelics (+), type: snps|indels|both|any [both]\n");
fprintf(stderr, " --no-version do not append version and command line to the header\n");
fprintf(stderr, " -N, --do-not-normalize do not normalize indels (with -m or -c s)\n");
fprintf(stderr, " -s, --strict-filter when merging (-m+), merged site is PASS only if all sites being merged PASS\n");
fprintf(stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
- fprintf(stderr, " --threads <int> number of extra (de)compression threads [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(stderr, " -w, --site-win <int> buffer for sorting lines which changed position during realignment [1000]\n");
fprintf(stderr, "\n");
+ fprintf(stderr, "Examples:\n");
+ fprintf(stderr, " # normalize and left-align indels\n");
+ fprintf(stderr, " bcftools norm -f ref.fa in.vcf\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, " # split multi-allelic sites\n");
+ fprintf(stderr, " bcftools norm -m- in.vcf\n");
+ fprintf(stderr, "\n");
exit(1);
}
static struct option loptions[] =
{
{"help",no_argument,NULL,'h'},
+ {"force",no_argument,NULL,7},
{"fasta-ref",required_argument,NULL,'f'},
{"do-not-normalize",no_argument,NULL,'N'},
{"multiallelics",required_argument,NULL,'m'},
else if ( !strcmp("all",optarg) ) args->rmdup = BCF_SR_PAIR_ANY;
else if ( !strcmp("any",optarg) ) args->rmdup = BCF_SR_PAIR_ANY;
else if ( !strcmp("none",optarg) ) args->rmdup = BCF_SR_PAIR_EXACT;
+ else if ( !strcmp("exact",optarg) ) args->rmdup = BCF_SR_PAIR_EXACT;
else error("The argument to -d not recognised: %s\n", optarg);
break;
case 'm':
break;
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
+ case 7 : args->force = 1; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
else fname = argv[optind];
if ( !args->ref_fname && !args->mrows_op && !args->rmdup ) error("Expected -f, -m, -D or -d option\n");
- if ( !args->ref_fname && args->check_ref&CHECK_REF_FIX ) error("Expected --fasta-ref with --check-ref s\n");
+ if ( !args->check_ref && args->ref_fname ) args->check_ref = CHECK_REF_EXIT;
+ if ( args->check_ref && !args->ref_fname ) error("Expected --fasta-ref with --check-ref\n");
if ( args->region )
{
}
if ( bcf_sr_set_threads(args->files, args->n_threads)<0 ) error("Failed to create threads\n");
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
if ( args->mrows_op&MROWS_SPLIT && args->rmdup ) error("Cannot combine -D and -m-\n");
init_data(args);
normalize_vcf(args);
/* vcfnorm.c -- Left-align and normalize indels.
- Copyright (C) 2013-2017 Genome Research Ltd.
+ Copyright (C) 2013-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
+#include <inttypes.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/faidx.h>
#include "bcftools.h"
#include "rbuf.h"
-#define CHECK_REF_EXIT 0
-#define CHECK_REF_WARN 1
-#define CHECK_REF_SKIP 2
-#define CHECK_REF_FIX 4
+#define CHECK_REF_EXIT 1
+#define CHECK_REF_WARN 2
+#define CHECK_REF_SKIP 4
+#define CHECK_REF_FIX 8
#define MROWS_SPLIT 1
#define MROWS_MERGE 2
char *ref, *alt;
void *hash;
}
+cmpals1_t;
+
+typedef struct
+{
+ cmpals1_t *cmpals;
+ int ncmpals, mcmpals;
+}
cmpals_t;
typedef struct
int aln_win; // the realignment window size (maximum repeat size)
bcf_srs_t *files; // using the synced reader only for -r option
bcf_hdr_t *hdr;
- cmpals_t *cmpals;
- int ncmpals, mcmpals;
+ cmpals_t cmpals_in, cmpals_out;
faidx_t *fai;
struct { int tot, set, swap; } nref;
char **argv, *output_fname, *ref_fname, *vcf_fname, *region, *targets;
int argc, rmdup, output_type, n_threads, check_ref, strict_filter, do_indels;
int nchanged, nskipped, nsplit, ntotal, mrows_op, mrows_collapse, parsimonious;
- int record_cmd_line;
+ int record_cmd_line, force, force_warned;
}
args_t;
}
char *ref = faidx_fetch_seq(args->fai, (char*)bcf_seqname(args->hdr,line), line->pos, line->pos+maxlen-1, &len);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ if ( !ref ) error("faidx_fetch_seq failed at %s:%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
replace_iupac_codes(ref,len);
args->nref.tot++;
int i, j, nals = line->n_allele, nals_ori = line->n_allele;
for (i=1, j=1; i<line->n_allele; i++)
{
- if ( strcmp(line->d.allele[0],line->d.allele[i]) )
+ if ( strcasecmp(line->d.allele[0],line->d.allele[i]) )
{
args->tmp_arr1[i] = j++;
continue;
// Sanity check REF
int i, nref, reflen = strlen(line->d.allele[0]);
char *ref = faidx_fetch_seq(args->fai, (char*)args->hdr->id[BCF_DT_CTG][line->rid].key, line->pos, line->pos+reflen-1, &nref);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", args->hdr->id[BCF_DT_CTG][line->rid].key, line->pos+1);
+ if ( !ref ) error("faidx_fetch_seq failed at %s:%"PRId64"\n", args->hdr->id[BCF_DT_CTG][line->rid].key, (int64_t) line->pos+1);
seq_to_upper(ref,0);
replace_iupac_codes(ref,nref); // any non-ACGT character in fasta ref is replaced with N
if ( has_non_acgtn(line->d.allele[0],reflen) )
{
if ( args->check_ref==CHECK_REF_EXIT )
- error("Non-ACGTN reference allele at %s:%d .. REF_SEQ:'%s' vs VCF:'%s'\n", bcf_seqname(args->hdr,line),line->pos+1,ref,line->d.allele[0]);
+ error("Non-ACGTN reference allele at %s:%"PRId64" .. REF_SEQ:'%s' vs VCF:'%s'\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,ref,line->d.allele[0]);
if ( args->check_ref & CHECK_REF_WARN )
- fprintf(bcftools_stderr,"NON_ACGTN_REF\t%s\t%d\t%s\n", bcf_seqname(args->hdr,line),line->pos+1,line->d.allele[0]);
+ fprintf(bcftools_stderr,"NON_ACGTN_REF\t%s\t%"PRId64"\t%s\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,line->d.allele[0]);
free(ref);
return ERR_REF_MISMATCH;
}
if ( strcasecmp(ref,line->d.allele[0]) )
{
if ( args->check_ref==CHECK_REF_EXIT )
- error("Reference allele mismatch at %s:%d .. REF_SEQ:'%s' vs VCF:'%s'\n", bcf_seqname(args->hdr,line),line->pos+1,ref,line->d.allele[0]);
+ error("Reference allele mismatch at %s:%"PRId64" .. REF_SEQ:'%s' vs VCF:'%s'\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,ref,line->d.allele[0]);
if ( args->check_ref & CHECK_REF_WARN )
- fprintf(bcftools_stderr,"REF_MISMATCH\t%s\t%d\t%s\n", bcf_seqname(args->hdr,line),line->pos+1,line->d.allele[0]);
+ fprintf(bcftools_stderr,"REF_MISMATCH\t%s\t%"PRId64"\t%s\t%s\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,line->d.allele[0],ref);
free(ref);
return ERR_REF_MISMATCH;
}
if ( has_non_acgtn(line->d.allele[i],line->shared.l) )
{
if ( args->check_ref==CHECK_REF_EXIT )
- error("Non-ACGTN alternate allele at %s:%d .. REF_SEQ:'%s' vs VCF:'%s'\n", bcf_seqname(args->hdr,line),line->pos+1,ref,line->d.allele[i]);
+ error("Non-ACGTN alternate allele at %s:%"PRId64" .. VCF:'%s'\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,line->d.allele[i]);
if ( args->check_ref & CHECK_REF_WARN )
- fprintf(bcftools_stderr,"NON_ACGTN_ALT\t%s\t%d\t%s\n", bcf_seqname(args->hdr,line),line->pos+1,line->d.allele[i]);
+ fprintf(bcftools_stderr,"NON_ACGTN_ALT\t%s\t%"PRId64"\t%s\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1,line->d.allele[i]);
return ERR_REF_MISMATCH;
}
kputs(line->d.allele[i], &als[i]);
seq_to_upper(als[i].s,0);
- if ( i>0 && als[i].l==als[0].l && !strcmp(als[0].s,als[i].s) ) return ERR_DUP_ALLELE;
+ if ( i>0 && als[i].l==als[0].l && !strcasecmp(als[0].s,als[i].s) ) return ERR_DUP_ALLELE;
}
// trim from right
int min_len = als[0].l;
for (i=1; i<line->n_allele; i++)
{
- if ( als[0].s[ als[0].l-1 ]!=als[i].s[ als[i].l-1 ] ) break;
+ if ( toupper(als[0].s[ als[0].l-1 ])!=toupper(als[i].s[ als[i].l-1 ]) ) break;
if ( als[i].l < min_len ) min_len = als[i].l;
}
if ( i!=line->n_allele ) break; // there are differences, cannot be trimmed
int npad = line->pos >= args->aln_win ? args->aln_win : line->pos;
free(ref);
ref = faidx_fetch_seq(args->fai, (char*)args->hdr->id[BCF_DT_CTG][line->rid].key, line->pos-npad, line->pos-1, &nref);
- if ( !ref ) error("faidx_fetch_seq failed at %s:%d\n", args->hdr->id[BCF_DT_CTG][line->rid].key, line->pos-npad+1);
+ if ( !ref ) error("faidx_fetch_seq failed at %s:%"PRId64"\n", args->hdr->id[BCF_DT_CTG][line->rid].key, (int64_t) line->pos-npad+1);
replace_iupac_codes(ref,nref);
for (i=0; i<line->n_allele; i++)
{
// Have the alleles changed?
als[0].s[ als[0].l ] = 0; // in order for strcmp to work
- if ( ori_pos==line->pos && !strcmp(line->d.allele[0],als[0].s) ) return ERR_OK;
+ if ( ori_pos==line->pos && !strcasecmp(line->d.allele[0],als[0].s) ) return ERR_OK;
// Create new block of alleles and update
args->tmp_als_str.l = 0;
if ( len==BCF_VL_A ) \
{ \
if ( ret!=src->n_allele-1 ) \
- error("Error: wrong number of fields in INFO/%s at %s:%d, expected %d, found %d\n", \
- tag,bcf_seqname(args->hdr,src),src->pos+1,src->n_allele-1,ret); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(bcftools_stderr, \
+ "Warning: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele-1,ret); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_info_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele-1,ret); \
+ } \
bcf_update_info_##type(args->hdr,dst,tag,vals+ialt,1); \
} \
else if ( len==BCF_VL_R ) \
{ \
if ( ret!=src->n_allele ) \
- error("Error: wrong number of fields in INFO/%s at %s:%d, expected %d, found %d\n", \
- tag,bcf_seqname(args->hdr,src),src->pos+1,src->n_allele,ret); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(bcftools_stderr, \
+ "Warning: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele,ret); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_info_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele,ret); \
+ } \
if ( ialt!=0 ) vals[1] = vals[ialt+1]; \
bcf_update_info_##type(args->hdr,dst,tag,vals,2); \
} \
else if ( len==BCF_VL_G ) \
{ \
if ( ret!=src->n_allele*(src->n_allele+1)/2 ) \
- error("Error: wrong number of fields in INFO/%s at %s:%d, expected %d, found %d\n", \
- tag,bcf_seqname(args->hdr,src),src->pos+1,src->n_allele*(src->n_allele+1)/2,ret); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(bcftools_stderr, \
+ "Warning: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele*(src->n_allele+1)/2,ret); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_info_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error: wrong number of fields in INFO/%s at %s:%"PRId64", expected %d, found %d\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele*(src->n_allele+1)/2,ret); \
+ } \
if ( ialt!=0 ) \
{ \
vals[1] = vals[bcf_alleles2gt(0,ialt+1)]; \
if ( len==BCF_VL_A ) \
{ \
if ( nvals!=(src->n_allele-1)*nsmpl ) \
- error("Error: wrong number of fields in FMT/%s at %s:%d, expected %d, found %d\n", \
- tag,bcf_seqname(args->hdr,src),src->pos+1,(src->n_allele-1)*nsmpl,nvals); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(bcftools_stderr, \
+ "Warning: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d, found %d. Removing the field.\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,(src->n_allele-1)*nsmpl,nvals); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_format_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d, found %d\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,(src->n_allele-1)*nsmpl,nvals); \
+ } \
nvals /= nsmpl; \
type_t *src_vals = vals, *dst_vals = vals; \
for (i=0; i<nsmpl; i++) \
else if ( len==BCF_VL_R ) \
{ \
if ( nvals!=src->n_allele*nsmpl ) \
- error("Error: wrong number of fields in FMT/%s at %s:%d, expected %d, found %d\n", \
- tag,bcf_seqname(args->hdr,src),src->pos+1,src->n_allele*nsmpl,nvals); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(bcftools_stderr, \
+ "Warning: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d, found %d. Removing the field.\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,(src->n_allele-1)*nsmpl,nvals); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_format_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d, found %d\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele*nsmpl,nvals); \
+ } \
nvals /= nsmpl; \
type_t *src_vals = vals, *dst_vals = vals; \
for (i=0; i<nsmpl; i++) \
else if ( len==BCF_VL_G ) \
{ \
if ( nvals!=src->n_allele*(src->n_allele+1)/2*nsmpl && nvals!=src->n_allele*nsmpl ) \
- error("Error at %s:%d, the tag %s has wrong number of fields\n", bcf_seqname(args->hdr,src),src->pos+1,bcf_hdr_int2id(args->hdr,BCF_DT_ID,fmt->id)); \
+ { \
+ if ( args->force && !args->force_warned ) \
+ { \
+ fprintf(bcftools_stderr, \
+ "Warning: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d, found %d. Removing the field.\n" \
+ " (This warning is printed only once.)\n", \
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,(src->n_allele-1)*nsmpl,nvals); \
+ args->force_warned = 1; \
+ } \
+ if ( args->force ) \
+ { \
+ bcf_update_format_##type(args->hdr,dst,tag,NULL,0); \
+ return; \
+ } \
+ error("Error at %s:%"PRId64", the tag %s has wrong number of fields\n", bcf_seqname(args->hdr,src),(int64_t) src->pos+1,bcf_hdr_int2id(args->hdr,BCF_DT_ID,fmt->id)); \
+ } \
nvals /= nsmpl; \
int all_haploid = nvals==src->n_allele ? 1 : 0; \
type_t *src_vals = vals, *dst_vals = vals; \
{
const char *tag = bcf_hdr_int2id(args->hdr,BCF_DT_ID,fmt->id);
int ret = bcf_get_format_char(args->hdr,src,tag,&args->tmp_arr1,&args->ntmp_arr1);
+ if ( !ret ) return; // all values can be empty, leave out the tag, no need to panic
assert( ret>0 );
kstring_t str;
if ( *se==',' ) nfields++;
se++;
}
+ if ( nfields==1 && se-ptr==1 && *ptr=='.' ) continue; // missing value
if ( nfields!=src->n_allele*(src->n_allele+1)/2 && nfields!=src->n_allele )
- error("Error: wrong number of fields in FMT/%s at %s:%d, expected %d or %d, found %d\n",
- tag,bcf_seqname(args->hdr,src),src->pos+1,src->n_allele*(src->n_allele+1)/2,src->n_allele,nfields);
+ {
+ if ( args->force && !args->force_warned )
+ {
+ fprintf(bcftools_stderr,
+ "Warning: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d or %d, found %d. Removing the field.\n"
+ " (This warning is printed only once.)\n",
+ tag,bcf_seqname(args->hdr,src),(int64_t)src->pos+1,src->n_allele*(src->n_allele+1)/2,src->n_allele,nfields);
+ args->force_warned = 1;
+ }
+ if ( args->force )
+ {
+ bcf_update_format_char(args->hdr,dst,tag,NULL,0);
+ return;
+ }
+ error("Error: wrong number of fields in FMT/%s at %s:%"PRId64", expected %d or %d, found %d\n",
+ tag,bcf_seqname(args->hdr,src),(int64_t) src->pos+1,src->n_allele*(src->n_allele+1)/2,src->n_allele,nfields);
+ }
int len = 0;
if ( nfields==src->n_allele ) // haploid
if ( len==BCF_VL_A ) \
{ \
if (nvals_ori!=lines[0]->n_allele - 1) \
- error("vcfnorm: number of fields in first record at position %s:%d for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele-1); \
+ error("vcfnorm: number of fields in first record at position %s:%"PRId64" for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),(int64_t) lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele-1); \
int nvals = dst->n_allele - 1; \
ENLARGE_ARRAY(type_t,set_missing,args->tmp_arr1,args->ntmp_arr1,1,nvals_ori,nvals); \
vals = (type_t*) args->tmp_arr1; \
if (nvals2<0) continue; /* info tag does not exist in this record, skip */ \
args->ntmp_arr2 = ntmp2 * sizeof(type_t); \
if (nvals2!=lines[i]->n_allele-1) \
- error("vcfnorm: could not merge INFO tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge INFO tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals2 = (type_t*) args->tmp_arr2; \
for (k=0; k<nvals2; k++) \
{ \
else if ( len==BCF_VL_R ) \
{ \
if (nvals_ori!=lines[0]->n_allele) \
- error("vcfnorm: number of fields in first record at position %s:%d for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele); \
+ error("vcfnorm: number of fields in first record at position %s:%"PRId64" for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),(int64_t) lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele); \
int nvals = dst->n_allele; \
ENLARGE_ARRAY(type_t,set_missing,args->tmp_arr1,args->ntmp_arr1,1,nvals_ori,nvals); \
vals = (type_t*) args->tmp_arr1; \
if (nvals2<0) continue; /* info tag does not exist in this record, skip */ \
args->ntmp_arr2 = ntmp2 * sizeof(type_t); \
if (nvals2!=lines[i]->n_allele) \
- error("vcfnorm: could not merge INFO tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge INFO tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals2 = (type_t*) args->tmp_arr2; \
for (k=0; k<nvals2; k++) \
{ \
/* expecting diploid gt in INFO */ \
if (nvals_ori!=lines[0]->n_allele*(lines[0]->n_allele+1)/2) { \
fprintf(bcftools_stderr, "todo: merge Number=G INFO fields for haploid sites\n"); \
- error("vcfnorm: number of fields in first record at position %s:%d for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele*(lines[0]->n_allele+1)/2); \
+ error("vcfnorm: number of fields in first record at position %s:%"PRId64" for INFO tag %s not as expected [found: %d vs expected:%d]\n", bcf_seqname(args->hdr,lines[0]),(int64_t) lines[0]->pos+1, tag, nvals_ori, lines[0]->n_allele*(lines[0]->n_allele+1)/2); \
} \
int nvals = dst->n_allele*(dst->n_allele+1)/2; \
ENLARGE_ARRAY(type_t,set_missing,args->tmp_arr1,args->ntmp_arr1,1,nvals_ori,nvals); \
if (nvals2<0) continue; /* info tag does not exist in this record, skip */ \
args->ntmp_arr2 = ntmp2 * sizeof(type_t); \
if (nvals2!=lines[i]->n_allele*(lines[i]->n_allele+1)/2) \
- error("vcfnorm: could not merge INFO tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge INFO tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals2 = (type_t*) args->tmp_arr2; \
int ia,ib; \
k = 0; \
int ngts2 = bcf_get_genotypes(args->hdr,lines[i],&args->tmp_arr2,&ntmp2);
args->ntmp_arr2 = ntmp2 * 4;
ngts2 /= nsmpl;
- if ( ngts!=ngts2 ) error("Error at %s:%d: cannot combine diploid with haploid genotype\n", bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1);
+ if ( ngts!=ngts2 ) error("Error at %s:%"PRId64": cannot combine diploid with haploid genotype\n", bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1);
int32_t *gt = (int32_t*) args->tmp_arr1;
int32_t *gt2 = (int32_t*) args->tmp_arr2;
else
{
int ial = bcf_gt_allele(gt2[k]);
- if ( ial>=args->maps[i].nals ) error("Error at %s:%d: incorrect allele index %d\n",bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1,ial);
+ if ( ial>=args->maps[i].nals ) error("Error at %s:%"PRId64": incorrect allele index %d\n",bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1,ial);
gt[k] = bcf_gt_unphased( args->maps[i].map[ial] ) | bcf_gt_is_phased(gt[k]);
}
}
args->ntmp_arr2 = ntmp2 * sizeof(type_t); \
nvals2 /= nsmpl; \
if (nvals2!=lines[i]->n_allele-1) \
- error("vcfnorm: could not merge FORMAT tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge FORMAT tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals = (type_t*) args->tmp_arr1; \
vals2 = (type_t*) args->tmp_arr2; \
for (j=0; j<nsmpl; j++) \
args->ntmp_arr2 = ntmp2 * sizeof(type_t); \
nvals2 /= nsmpl; \
if (nvals2!=lines[i]->n_allele) \
- error("vcfnorm: could not merge FORMAT tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge FORMAT tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals = (type_t*) args->tmp_arr1; \
vals2 = (type_t*) args->tmp_arr2; \
for (j=0; j<nsmpl; j++) \
int ndiploid = lines[i]->n_allele*(lines[i]->n_allele+1)/2; \
int line_diploid = nvals2==ndiploid ? 1 : 0; \
if (!(nvals2==1 || nvals2==lines[i]->n_allele || nvals2==lines[i]->n_allele*(lines[i]->n_allele+1)/2)) \
- error("vcfnorm: could not merge FORMAT tag %s at position %s:%d\n", tag, bcf_seqname(args->hdr,lines[i]),lines[i]->pos+1); \
+ error("vcfnorm: could not merge FORMAT tag %s at position %s:%"PRId64"\n", tag, bcf_seqname(args->hdr,lines[i]),(int64_t) lines[i]->pos+1); \
vals = (type_t*) args->tmp_arr1; \
vals2 = (type_t*) args->tmp_arr2; \
for (j=0; j<nsmpl; j++) \
haploid[i] = 0;
nfields = dst->n_allele*(dst->n_allele+1)/2;
}
- else error("The field %s at %s:%d neither diploid nor haploid?\n", tag,bcf_seqname(args->hdr,dst),dst->pos+1);
+ else error("The field %s at %s:%"PRId64" neither diploid nor haploid?\n", tag,bcf_seqname(args->hdr,dst),(int64_t) dst->pos+1);
kstring_t *tmp = &args->tmp_str[i];
kputc('.',tmp);
args->maps[i].nals = lines[i]->n_allele;
hts_expand(int,args->maps[i].nals,args->maps[i].mals,args->maps[i].map);
args->als = merge_alleles(lines[i]->d.allele, lines[i]->n_allele, args->maps[i].map, args->als, &args->nals, &args->mals);
- if ( !args->als ) error("Failed to merge alleles at %s:%d\n", bcf_seqname(args->hdr,dst),dst->pos+1);
+ if ( !args->als ) error("Failed to merge alleles at %s:%"PRId64"\n", bcf_seqname(args->hdr,dst),(int64_t) dst->pos+1);
}
bcf_update_alleles(args->hdr, dst, (const char**)args->als, args->nals);
for (i=0; i<args->nals; i++)
}
return NULL;
}
-static void cmpals_add(args_t *args, bcf1_t *rec)
+static void cmpals_add(cmpals_t *ca, bcf1_t *rec)
{
- args->ncmpals++;
- hts_expand0(cmpals_t, args->ncmpals, args->mcmpals, args->cmpals);
- cmpals_t *cmpals = args->cmpals + args->ncmpals - 1;
+ ca->ncmpals++;
+ hts_expand0(cmpals1_t, ca->ncmpals, ca->mcmpals, ca->cmpals);
+ cmpals1_t *cmpals = ca->cmpals + ca->ncmpals - 1;
free(cmpals->ref);
cmpals->ref = strdup(rec->d.allele[0]);
cmpals->n = rec->n_allele;
khash_str2int_inc(cmpals->hash, strdup(rec->d.allele[i]));
}
}
-static int cmpals_match(args_t *args, bcf1_t *rec)
+static int cmpals_match(cmpals_t *ca, bcf1_t *rec)
{
int i, j;
- for (i=0; i<args->ncmpals; i++)
+ for (i=0; i<ca->ncmpals; i++)
{
- cmpals_t *cmpals = args->cmpals + i;
+ cmpals1_t *cmpals = ca->cmpals + i;
if ( rec->n_allele != cmpals->n ) continue;
// NB. assuming both are normalized
- if ( strcmp(rec->d.allele[0], cmpals->ref) ) continue;
+ if ( strcasecmp(rec->d.allele[0], cmpals->ref) ) continue;
// the most frequent case
if ( rec->n_allele==2 )
{
- if ( strcmp(rec->d.allele[1], cmpals->alt) ) continue;
+ if ( strcasecmp(rec->d.allele[1], cmpals->alt) ) continue;
return 1;
}
if ( j<rec->n_allele ) continue;
return 1;
}
- cmpals_add(args, rec);
return 0;
}
-static void cmpals_reset(args_t *args) { args->ncmpals = 0; }
-static void cmpals_destroy(args_t *args)
+static void cmpals_reset(cmpals_t *ca) { ca->ncmpals = 0; }
+static void cmpals_destroy(cmpals_t *ca)
{
int i;
- for (i=0; i<args->mcmpals; i++)
+ for (i=0; i<ca->mcmpals; i++)
{
- cmpals_t *cmpals = args->cmpals + i;
+ cmpals1_t *cmpals = ca->cmpals + i;
free(cmpals->ref);
free(cmpals->alt);
if ( cmpals->hash ) khash_str2int_destroy_free(cmpals->hash);
}
- free(args->cmpals);
+ free(ca->cmpals);
}
static void flush_buffer(args_t *args, htsFile *file, int n)
{
if ( mrows_ready_to_flush(args, args->lines[k]) )
{
- while ( (line=mrows_flush(args)) ) bcf_write1(file, args->hdr, line);
+ while ( (line=mrows_flush(args)) )
+ if ( bcf_write1(file, args->hdr, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
int merge = 1;
if ( args->mrows_collapse!=COLLAPSE_BOTH && args->mrows_collapse!=COLLAPSE_ANY )
if ( args->rmdup & BCF_SR_PAIR_ANY ) continue; // rmdup by position only
if ( args->rmdup & BCF_SR_PAIR_SNPS && line_type&(VCF_SNP|VCF_MNP) && prev_type&(VCF_SNP|VCF_MNP) ) continue;
if ( args->rmdup & BCF_SR_PAIR_INDELS && line_type&(VCF_INDEL) && prev_type&(VCF_INDEL) ) continue;
- if ( args->rmdup & BCF_SR_PAIR_EXACT && cmpals_match(args, args->lines[k]) ) continue;
+ if ( args->rmdup & BCF_SR_PAIR_EXACT && cmpals_match(&args->cmpals_out, args->lines[k]) ) continue;
}
else
{
prev_rid = args->lines[k]->rid;
prev_pos = args->lines[k]->pos;
prev_type = 0;
- if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_reset(args);
+ if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_reset(&args->cmpals_out);
}
prev_type |= line_type;
- if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_add(args, args->lines[k]);
+ if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_add(&args->cmpals_out, args->lines[k]);
}
- bcf_write1(file, args->hdr, args->lines[k]);
+ if ( bcf_write1(file, args->hdr, args->lines[k])!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
if ( args->mrows_op==MROWS_MERGE && !args->rbuf.n )
{
- while ( (line=mrows_flush(args)) ) bcf_write1(file, args->hdr, line);
+ while ( (line=mrows_flush(args)) )
+ if ( bcf_write1(file, args->hdr, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
}
static void destroy_data(args_t *args)
{
- cmpals_destroy(args);
+ cmpals_destroy(&args->cmpals_in);
+ cmpals_destroy(&args->cmpals_out);
int i;
for (i=0; i<args->rbuf.m; i++)
if ( args->lines[i] ) bcf_destroy1(args->lines[i]);
if ( args->check_ref & CHECK_REF_FIX )
fix_dup_alt(args, line);
else if ( args->check_ref==CHECK_REF_EXIT )
- error("Duplicate alleles at %s:%d; run with -cw to turn the error into warning or with -cs to fix.\n", bcf_seqname(args->hdr,line),line->pos+1);
+ error("Duplicate alleles at %s:%"PRId64"; run with -cw to turn the error into warning or with -cs to fix.\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
else if ( args->check_ref & CHECK_REF_WARN )
- fprintf(bcftools_stderr,"ALT_DUP\t%s\t%d\n", bcf_seqname(args->hdr,line),line->pos+1);
+ fprintf(bcftools_stderr,"ALT_DUP\t%s\t%"PRId64"\n", bcf_seqname(args->hdr,line),(int64_t) line->pos+1);
}
}
}
if ( args->n_threads )
hts_set_opt(out, HTS_OPT_THREAD_POOL, args->files->p);
if (args->record_cmd_line) bcf_hdr_append_version(args->hdr, args->argc, args->argv, "bcftools_norm");
- bcf_hdr_write(out, args->hdr);
+ if ( bcf_hdr_write(out, args->hdr)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
int prev_rid = -1, prev_pos = -1, prev_type = 0;
while ( bcf_sr_next_line(args->files) )
if ( args->rmdup & BCF_SR_PAIR_ANY ) continue; // rmdup by position only
if ( args->rmdup & BCF_SR_PAIR_SNPS && line_type&(VCF_SNP|VCF_MNP) && prev_type&(VCF_SNP|VCF_MNP) ) continue;
if ( args->rmdup & BCF_SR_PAIR_INDELS && line_type&(VCF_INDEL) && prev_type&(VCF_INDEL) ) continue;
- if ( args->rmdup & BCF_SR_PAIR_EXACT && cmpals_match(args, line) ) continue;
+ if ( args->rmdup & BCF_SR_PAIR_EXACT && cmpals_match(&args->cmpals_in, line) ) continue;
}
else
{
prev_rid = line->rid;
prev_pos = line->pos;
prev_type = 0;
- if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_reset(args);
+ if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_reset(&args->cmpals_in);
}
prev_type |= line_type;
- if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_add(args, line);
+ if ( args->rmdup & BCF_SR_PAIR_EXACT ) cmpals_add(&args->cmpals_in, line);
}
// still on the same chromosome?
if ( j>0 ) flush_buffer(args, out, j);
}
flush_buffer(args, out, args->rbuf.n);
- hts_close(out);
+ if ( hts_close(out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
fprintf(bcftools_stderr,"Lines total/split/realigned/skipped:\t%d/%d/%d/%d\n", args->ntotal,args->nsplit,args->nchanged,args->nskipped);
if ( args->check_ref & CHECK_REF_FIX )
fprintf(bcftools_stderr, "Options:\n");
fprintf(bcftools_stderr, " -c, --check-ref <e|w|x|s> check REF alleles and exit (e), warn (w), exclude (x), or set (s) bad sites [e]\n");
fprintf(bcftools_stderr, " -D, --remove-duplicates remove duplicate lines of the same type.\n");
- fprintf(bcftools_stderr, " -d, --rm-dup <type> remove duplicate snps|indels|both|all|none\n");
- fprintf(bcftools_stderr, " -f, --fasta-ref <file> reference sequence (MANDATORY)\n");
+ fprintf(bcftools_stderr, " -d, --rm-dup <type> remove duplicate snps|indels|both|all|exact\n");
+ fprintf(bcftools_stderr, " -f, --fasta-ref <file> reference sequence\n");
+ fprintf(bcftools_stderr, " --force try to proceed even if malformed tags are encountered. Experimental, use at your own risk\n");
fprintf(bcftools_stderr, " -m, --multiallelics <-|+>[type] split multiallelics (-) or join biallelics (+), type: snps|indels|both|any [both]\n");
fprintf(bcftools_stderr, " --no-version do not append version and command line to the header\n");
fprintf(bcftools_stderr, " -N, --do-not-normalize do not normalize indels (with -m or -c s)\n");
fprintf(bcftools_stderr, " -s, --strict-filter when merging (-m+), merged site is PASS only if all sites being merged PASS\n");
fprintf(bcftools_stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(bcftools_stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra (de)compression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(bcftools_stderr, " -w, --site-win <int> buffer for sorting lines which changed position during realignment [1000]\n");
fprintf(bcftools_stderr, "\n");
+ fprintf(bcftools_stderr, "Examples:\n");
+ fprintf(bcftools_stderr, " # normalize and left-align indels\n");
+ fprintf(bcftools_stderr, " bcftools norm -f ref.fa in.vcf\n");
+ fprintf(bcftools_stderr, "\n");
+ fprintf(bcftools_stderr, " # split multi-allelic sites\n");
+ fprintf(bcftools_stderr, " bcftools norm -m- in.vcf\n");
+ fprintf(bcftools_stderr, "\n");
exit(1);
}
static struct option loptions[] =
{
{"help",no_argument,NULL,'h'},
+ {"force",no_argument,NULL,7},
{"fasta-ref",required_argument,NULL,'f'},
{"do-not-normalize",no_argument,NULL,'N'},
{"multiallelics",required_argument,NULL,'m'},
else if ( !strcmp("all",optarg) ) args->rmdup = BCF_SR_PAIR_ANY;
else if ( !strcmp("any",optarg) ) args->rmdup = BCF_SR_PAIR_ANY;
else if ( !strcmp("none",optarg) ) args->rmdup = BCF_SR_PAIR_EXACT;
+ else if ( !strcmp("exact",optarg) ) args->rmdup = BCF_SR_PAIR_EXACT;
else error("The argument to -d not recognised: %s\n", optarg);
break;
case 'm':
break;
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
+ case 7 : args->force = 1; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
else fname = argv[optind];
if ( !args->ref_fname && !args->mrows_op && !args->rmdup ) error("Expected -f, -m, -D or -d option\n");
- if ( !args->ref_fname && args->check_ref&CHECK_REF_FIX ) error("Expected --fasta-ref with --check-ref s\n");
+ if ( !args->check_ref && args->ref_fname ) args->check_ref = CHECK_REF_EXIT;
+ if ( args->check_ref && !args->ref_fname ) error("Expected --fasta-ref with --check-ref\n");
if ( args->region )
{
}
if ( bcf_sr_set_threads(args->files, args->n_threads)<0 ) error("Failed to create threads\n");
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
if ( args->mrows_op&MROWS_SPLIT && args->rmdup ) error("Cannot combine -D and -m-\n");
init_data(args);
normalize_vcf(args);
#include <htslib/synced_bcf_reader.h>
#include <htslib/kseq.h>
#include <htslib/khash_str2int.h>
+#ifdef _WIN32
+#include <windows.h>
+#else
#include <dlfcn.h>
+#endif
#include "bcftools.h"
#include "vcmp.h"
#include "filter.h"
{
while (1)
{
- size_t len = strcspn(path, ":");
+ size_t len = strcspn(path, HTS_PATH_SEPARATOR_STR);
if ( len == 0 )
{
}
path += len;
- if ( *path == ':' ) path++;
+ if ( *path == HTS_PATH_SEPARATOR_CHAR ) path++;
else break;
}
}
void *handle;
char *tmp;
- if ( fname[0]!='/' ) // not an absolute path
+ int is_absolute_path = 0;
+#ifdef _WIN32
+ // Windows accepts both forward slash (/) and backslash (\) as folder separator
+ // and can have any path prefixed by the drive letter and a colon (:).
+ if ( fname[0]=='/' || fname[0]=='\\') is_absolute_path = 1;
+ else if ( fname[0] && fname[1]==':' && (fname[2]=='/' || fname[2]=='\\') ) is_absolute_path = 1;
+#else
+ if ( fname[0]=='/' ) is_absolute_path = 1;
+#endif
+ if ( !is_absolute_path )
{
int i;
for (i=0; i<args->nplugin_paths; i++)
{
- tmp = msprintf("%s/%s%s", args->plugin_paths[i], fname, PLUGIN_EXT);
+ tmp = msprintf("%s/%s%s", args->plugin_paths[i], fname, PLUGIN_EXT);
+#ifdef _WIN32
+ handle = LoadLibraryA(tmp);
+#else
handle = dlopen(tmp, RTLD_NOW); // valgrind complains about unfreed memory, not our problem though
+#endif
if ( args->verbose > 1 )
{
- if ( !handle ) fprintf(stderr,"%s:\n\tdlopen .. %s\n", tmp,dlerror());
- else fprintf(stderr,"%s:\n\tdlopen .. ok\n", tmp);
+ if ( !handle )
+#ifdef _WIN32
+ fprintf(stderr,"%s:\n\tLoadLibraryA .. %lu\n", tmp, GetLastError());
+#else
+ fprintf(stderr,"%s:\n\tdlopen .. %s\n", tmp, dlerror());
+#endif
+ else fprintf(stderr,"%s:\n\tplugin open .. ok\n", tmp);
}
free(tmp);
if ( handle ) return handle;
}
}
+#ifdef _WIN32
+ handle = LoadLibraryA(fname);
+#else
handle = dlopen(fname, RTLD_NOW);
+#endif
if ( args->verbose > 1 )
{
- if ( !handle ) fprintf(stderr,"%s:\n\tdlopen .. %s\n", fname,dlerror());
- else fprintf(stderr,"%s:\n\tdlopen .. ok\n", fname);
+ if ( !handle )
+#ifdef _WIN32
+ fprintf(stderr,"%s:\n\tLoadLibraryA .. %lu\n", fname, GetLastError());
+#else
+ fprintf(stderr,"%s:\n\tdlopen .. %s\n", fname, dlerror());
+#endif
+ else fprintf(stderr,"%s:\n\tplugin open .. ok\n", fname);
}
return handle;
return -1;
}
+#ifdef _WIN32
+ plugin->init = (dl_init_f) GetProcAddress(plugin->handle, "init");
+ if ( plugin->init && args->verbose > 1 ) fprintf(stderr,"\tinit .. ok\n");
+
+ plugin->run = (dl_run_f) GetProcAddress(plugin->handle, "run");
+ if ( plugin->run && args->verbose > 1 ) fprintf(stderr,"\trun .. ok\n");
+
+ if ( !plugin->init && !plugin->run )
+ {
+ if ( exit_on_error ) error("Could not initialize %s, neither run or init found \n", plugin->name);
+ else if ( args->verbose > 1 ) fprintf(stderr,"\tinit/run .. not found\n");
+ return -1;
+ }
+
+ plugin->version = (dl_version_f) GetProcAddress(plugin->handle, "version");
+ if ( !plugin->version )
+ {
+ if ( exit_on_error ) error("Could not initialize %s: version string not found\n", plugin->name);
+ else if ( args->verbose > 1 ) fprintf(stderr,"\tversion .. not found\n");
+ return -1;
+ }
+
+ plugin->about = (dl_about_f) GetProcAddress(plugin->handle, "about");
+ if ( !plugin->about )
+ {
+ if ( exit_on_error ) error("Could not initialize %s: about string not found\n", plugin->name);
+ return -1;
+ }
+
+ plugin->usage = (dl_about_f) GetProcAddress(plugin->handle, "usage");
+ if ( !plugin->usage )
+ plugin->usage = plugin->about;
+
+ if ( plugin->run ) return 0;
+
+ plugin->process = (dl_process_f) GetProcAddress(plugin->handle, "process");
+ if ( !plugin->process )
+ {
+ if ( exit_on_error ) error("Could not initialize %s: process method not found\n", plugin->name);
+ return -1;
+ }
+
+ plugin->destroy = (dl_destroy_f) GetProcAddress(plugin->handle, "destroy");
+ if ( !plugin->destroy )
+ {
+ if ( exit_on_error ) error("Could not initialize %s: destroy method not found\n", plugin->name);
+ return -1;
+ }
+#else
dlerror();
plugin->init = (dl_init_f) dlsym(plugin->handle, "init");
char *ret = dlerror();
if ( exit_on_error ) error("Could not initialize %s: %s\n", plugin->name, ret);
return -1;
}
+#endif
return 0;
}
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
if ( args->n_threads ) hts_set_threads(args->out_fh, args->n_threads);
- bcf_hdr_write(args->out_fh, args->hdr_out);
+ if ( bcf_hdr_write(args->out_fh, args->hdr_out)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
}
{
free(args->plugin.name);
if ( args->plugin.destroy ) args->plugin.destroy();
+#ifdef _WIN32
+ FreeLibrary(args->plugin.handle);
+#else
dlclose(args->plugin.handle);
+#endif
if ( args->hdr_out ) bcf_hdr_destroy(args->hdr_out);
if ( args->nplugin_paths>0 )
{
}
if ( args->filter )
filter_destroy(args->filter);
- if (args->out_fh) hts_close(args->out_fh);
+ if (args->out_fh && hts_close(args->out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
}
static void usage(args_t *args)
fprintf(stderr, " --no-version do not append version and command line to the header\n");
fprintf(stderr, " -o, --output <file> write output to a file [standard output]\n");
fprintf(stderr, " -O, --output-type <type> 'b' compressed BCF; 'u' uncompressed BCF; 'z' compressed VCF; 'v' uncompressed VCF [v]\n");
- fprintf(stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(stderr, "Plugin options:\n");
fprintf(stderr, " -h, --help list plugin's options\n");
fprintf(stderr, " -l, --list-plugins list available plugins. See BCFTOOLS_PLUGINS environment variable and man page for details\n");
char *fname = NULL;
if ( optind>=argc || argv[optind][0]=='-' )
{
- if ( !isatty(fileno((FILE *)stdin)) ) fname = "-"; // reading from stdin
- else usage(args);
args->plugin.argc = argc - optind + 1;
args->plugin.argv = argv + optind - 1;
+
+ if ( !isatty(fileno((FILE *)stdin)) ) fname = "-"; // reading from stdin
+ else if ( optind>=argc ) usage(args);
+ else
+ {
+ optind = 1;
+ init_plugin(args);
+ }
}
else
{
error("Failed to read the targets: %s\n", args->targets_list);
args->files->collapse |= COLLAPSE_SOME;
}
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
init_data(args);
while ( bcf_sr_next_line(args->files) )
if ( line )
{
if ( line->errcode ) error("[E::main_plugin] Unchecked error (%d), exiting\n",line->errcode);
- bcf_write1(args->out_fh, args->hdr_out, line);
+ if ( bcf_write1(args->out_fh, args->hdr_out, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
}
destroy_data(args);
#include <htslib/synced_bcf_reader.h>
#include <htslib/kseq.h>
#include <htslib/khash_str2int.h>
+#ifdef _WIN32
+#include <windows.h>
+#else
#include <dlfcn.h>
+#endif
#include "bcftools.h"
#include "vcmp.h"
#include "filter.h"
{
while (1)
{
- size_t len = strcspn(path, ":");
+ size_t len = strcspn(path, HTS_PATH_SEPARATOR_STR);
if ( len == 0 )
{
}
path += len;
- if ( *path == ':' ) path++;
+ if ( *path == HTS_PATH_SEPARATOR_CHAR ) path++;
else break;
}
}
void *handle;
char *tmp;
- if ( fname[0]!='/' ) // not an absolute path
+ int is_absolute_path = 0;
+#ifdef _WIN32
+ // Windows accepts both forward slash (/) and backslash (\) as folder separator
+ // and can have any path prefixed by the drive letter and a colon (:).
+ if ( fname[0]=='/' || fname[0]=='\\') is_absolute_path = 1;
+ else if ( fname[0] && fname[1]==':' && (fname[2]=='/' || fname[2]=='\\') ) is_absolute_path = 1;
+#else
+ if ( fname[0]=='/' ) is_absolute_path = 1;
+#endif
+ if ( !is_absolute_path )
{
int i;
for (i=0; i<args->nplugin_paths; i++)
{
- tmp = msprintf("%s/%s%s", args->plugin_paths[i], fname, PLUGIN_EXT);
+ tmp = msprintf("%s/%s%s", args->plugin_paths[i], fname, PLUGIN_EXT);
+#ifdef _WIN32
+ handle = LoadLibraryA(tmp);
+#else
handle = dlopen(tmp, RTLD_NOW); // valgrind complains about unfreed memory, not our problem though
+#endif
if ( args->verbose > 1 )
{
- if ( !handle ) fprintf(bcftools_stderr,"%s:\n\tdlopen .. %s\n", tmp,dlerror());
- else fprintf(bcftools_stderr,"%s:\n\tdlopen .. ok\n", tmp);
+ if ( !handle )
+#ifdef _WIN32
+ fprintf(bcftools_stderr,"%s:\n\tLoadLibraryA .. %lu\n", tmp, GetLastError());
+#else
+ fprintf(bcftools_stderr,"%s:\n\tdlopen .. %s\n", tmp, dlerror());
+#endif
+ else fprintf(bcftools_stderr,"%s:\n\tplugin open .. ok\n", tmp);
}
free(tmp);
if ( handle ) return handle;
}
}
+#ifdef _WIN32
+ handle = LoadLibraryA(fname);
+#else
handle = dlopen(fname, RTLD_NOW);
+#endif
if ( args->verbose > 1 )
{
- if ( !handle ) fprintf(bcftools_stderr,"%s:\n\tdlopen .. %s\n", fname,dlerror());
- else fprintf(bcftools_stderr,"%s:\n\tdlopen .. ok\n", fname);
+ if ( !handle )
+#ifdef _WIN32
+ fprintf(bcftools_stderr,"%s:\n\tLoadLibraryA .. %lu\n", fname, GetLastError());
+#else
+ fprintf(bcftools_stderr,"%s:\n\tdlopen .. %s\n", fname, dlerror());
+#endif
+ else fprintf(bcftools_stderr,"%s:\n\tplugin open .. ok\n", fname);
}
return handle;
return -1;
}
+#ifdef _WIN32
+ plugin->init = (dl_init_f) GetProcAddress(plugin->handle, "init");
+ if ( plugin->init && args->verbose > 1 ) fprintf(bcftools_stderr,"\tinit .. ok\n");
+
+ plugin->run = (dl_run_f) GetProcAddress(plugin->handle, "run");
+ if ( plugin->run && args->verbose > 1 ) fprintf(bcftools_stderr,"\trun .. ok\n");
+
+ if ( !plugin->init && !plugin->run )
+ {
+ if ( exit_on_error ) error("Could not initialize %s, neither run or init found \n", plugin->name);
+ else if ( args->verbose > 1 ) fprintf(bcftools_stderr,"\tinit/run .. not found\n");
+ return -1;
+ }
+
+ plugin->version = (dl_version_f) GetProcAddress(plugin->handle, "version");
+ if ( !plugin->version )
+ {
+ if ( exit_on_error ) error("Could not initialize %s: version string not found\n", plugin->name);
+ else if ( args->verbose > 1 ) fprintf(bcftools_stderr,"\tversion .. not found\n");
+ return -1;
+ }
+
+ plugin->about = (dl_about_f) GetProcAddress(plugin->handle, "about");
+ if ( !plugin->about )
+ {
+ if ( exit_on_error ) error("Could not initialize %s: about string not found\n", plugin->name);
+ return -1;
+ }
+
+ plugin->usage = (dl_about_f) GetProcAddress(plugin->handle, "usage");
+ if ( !plugin->usage )
+ plugin->usage = plugin->about;
+
+ if ( plugin->run ) return 0;
+
+ plugin->process = (dl_process_f) GetProcAddress(plugin->handle, "process");
+ if ( !plugin->process )
+ {
+ if ( exit_on_error ) error("Could not initialize %s: process method not found\n", plugin->name);
+ return -1;
+ }
+
+ plugin->destroy = (dl_destroy_f) GetProcAddress(plugin->handle, "destroy");
+ if ( !plugin->destroy )
+ {
+ if ( exit_on_error ) error("Could not initialize %s: destroy method not found\n", plugin->name);
+ return -1;
+ }
+#else
dlerror();
plugin->init = (dl_init_f) dlsym(plugin->handle, "init");
char *ret = dlerror();
if ( exit_on_error ) error("Could not initialize %s: %s\n", plugin->name, ret);
return -1;
}
+#endif
return 0;
}
args->out_fh = hts_open(args->output_fname,hts_bcf_wmode(args->output_type));
if ( args->out_fh == NULL ) error("Can't write to \"%s\": %s\n", args->output_fname, strerror(errno));
if ( args->n_threads ) hts_set_threads(args->out_fh, args->n_threads);
- bcf_hdr_write(args->out_fh, args->hdr_out);
+ if ( bcf_hdr_write(args->out_fh, args->hdr_out)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
}
{
free(args->plugin.name);
if ( args->plugin.destroy ) args->plugin.destroy();
+#ifdef _WIN32
+ FreeLibrary(args->plugin.handle);
+#else
dlclose(args->plugin.handle);
+#endif
if ( args->hdr_out ) bcf_hdr_destroy(args->hdr_out);
if ( args->nplugin_paths>0 )
{
}
if ( args->filter )
filter_destroy(args->filter);
- if (args->out_fh) hts_close(args->out_fh);
+ if (args->out_fh && hts_close(args->out_fh)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->output_fname);
}
static void usage(args_t *args)
fprintf(bcftools_stderr, " --no-version do not append version and command line to the header\n");
fprintf(bcftools_stderr, " -o, --output <file> write output to a file [standard output]\n");
fprintf(bcftools_stderr, " -O, --output-type <type> 'b' compressed BCF; 'u' uncompressed BCF; 'z' compressed VCF; 'v' uncompressed VCF [v]\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra output compression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(bcftools_stderr, "Plugin options:\n");
fprintf(bcftools_stderr, " -h, --help list plugin's options\n");
fprintf(bcftools_stderr, " -l, --list-plugins list available plugins. See BCFTOOLS_PLUGINS environment variable and man page for details\n");
char *fname = NULL;
if ( optind>=argc || argv[optind][0]=='-' )
{
- if ( !isatty(fileno((FILE *)stdin)) ) fname = "-"; // reading from stdin
- else usage(args);
args->plugin.argc = argc - optind + 1;
args->plugin.argv = argv + optind - 1;
+
+ if ( !isatty(fileno((FILE *)stdin)) ) fname = "-"; // reading from stdin
+ else if ( optind>=argc ) usage(args);
+ else
+ {
+ optind = 1;
+ init_plugin(args);
+ }
}
else
{
error("Failed to read the targets: %s\n", args->targets_list);
args->files->collapse |= COLLAPSE_SOME;
}
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
init_data(args);
while ( bcf_sr_next_line(args->files) )
if ( line )
{
if ( line->errcode ) error("[E::main_plugin] Unchecked error (%d), exiting\n",line->errcode);
- bcf_write1(args->out_fh, args->hdr_out, line);
+ if ( bcf_write1(args->out_fh, args->hdr_out, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->output_fname);
}
}
destroy_data(args);
if ( args->print_header )
{
convert_header(args->convert,&str);
- fwrite(str.s, str.l, 1, args->out);
+ if ( fwrite(str.s, str.l, 1, args->out)!=1 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out?args->fn_out:"standard output");
}
int i,max_convert_unpack = convert_max_unpack(args->convert);
str.l = 0;
convert_line(args->convert, line, &str);
- if ( str.l )
- fwrite(str.s, str.l, 1, args->out);
+ if ( str.l && fwrite(str.s, str.l, 1, args->out)!=1 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out?args->fn_out:"standard output");
}
if ( str.m ) free(str.s);
}
case 's': args->sample_list = optarg; break;
case 'S': args->sample_list = optarg; args->sample_is_file = 1; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
{
if ( !fname ) error("Missing the VCF file name\n");
args->files = bcf_sr_init();
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
list_columns(args);
bcf_sr_destroy(args->files);
free(args);
return 0;
}
- if ( !args->format_str ) usage();
+ if ( !args->format_str )
+ {
+ if ( argc==1 && !fname ) usage();
+ error("Error: Missing the --format option\n");
+ }
args->out = args->fn_out ? fopen(args->fn_out, "w") : stdout;
if ( !args->out ) error("%s: %s\n", args->fn_out,strerror(errno));
}
while ( fname )
{
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
fname = ++optind < argc ? argv[optind] : NULL;
}
init_data(args);
free(args->format_str);
destroy_data(args);
bcf_sr_destroy(args->files);
- fclose(args->out);
+ if ( fclose(args->out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fn_out);
free(args);
return 0;
}
if ( !bcf_sr_add_reader(args->files, argv[k]) ) error("Failed to open %s: %s\n", argv[k],bcf_sr_strerror(args->files->errnum));
init_data(args);
if ( i==0 )
+ {
prev_samples = copy_header(args->header, args->files->readers[0].header->samples, bcf_hdr_nsamples(args->files->readers[0].header));
+ prev_nsamples = bcf_hdr_nsamples(args->files->readers[0].header);
+ }
else
{
args->print_header = 0;
destroy_data(args);
bcf_sr_destroy(args->files);
}
- fclose(args->out);
+ if ( fclose(args->out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fn_out);;
destroy_list(fnames, nfiles);
destroy_list(prev_samples, prev_nsamples);
free(args->format_str);
if ( args->print_header )
{
convert_header(args->convert,&str);
- fwrite(str.s, str.l, 1, args->out);
+ if ( fwrite(str.s, str.l, 1, args->out)!=1 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out?args->fn_out:"standard output");
}
int i,max_convert_unpack = convert_max_unpack(args->convert);
str.l = 0;
convert_line(args->convert, line, &str);
- if ( str.l )
- fwrite(str.s, str.l, 1, args->out);
+ if ( str.l && fwrite(str.s, str.l, 1, args->out)!=1 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out?args->fn_out:"standard output");
}
if ( str.m ) free(str.s);
}
case 's': args->sample_list = optarg; break;
case 'S': args->sample_list = optarg; args->sample_is_file = 1; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
{
if ( !fname ) error("Missing the VCF file name\n");
args->files = bcf_sr_init();
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
list_columns(args);
bcf_sr_destroy(args->files);
free(args);
return 0;
}
- if ( !args->format_str ) usage();
+ if ( !args->format_str )
+ {
+ if ( argc==1 && !fname ) usage();
+ error("Error: Missing the --format option\n");
+ }
args->out = args->fn_out ? fopen(args->fn_out, "w") : bcftools_stdout;
if ( !args->out ) error("%s: %s\n", args->fn_out,strerror(errno));
}
while ( fname )
{
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
fname = ++optind < argc ? argv[optind] : NULL;
}
init_data(args);
free(args->format_str);
destroy_data(args);
bcf_sr_destroy(args->files);
- fclose(args->out);
+ if ( fclose(args->out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fn_out);
free(args);
return 0;
}
if ( !bcf_sr_add_reader(args->files, argv[k]) ) error("Failed to open %s: %s\n", argv[k],bcf_sr_strerror(args->files->errnum));
init_data(args);
if ( i==0 )
+ {
prev_samples = copy_header(args->header, args->files->readers[0].header->samples, bcf_hdr_nsamples(args->files->readers[0].header));
+ prev_nsamples = bcf_hdr_nsamples(args->files->readers[0].header);
+ }
else
{
args->print_header = 0;
destroy_data(args);
bcf_sr_destroy(args->files);
}
- fclose(args->out);
+ if ( fclose(args->out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fn_out);;
destroy_list(fnames, nfiles);
destroy_list(prev_samples, prev_nsamples);
free(args->format_str);
return mem;
}
+static inline int max255(int i)
+{
+ return i < 256 ? i : 255;
+}
+
static void init_data(args_t *args)
{
int i;
if ( !strncmp("GT,",args->estimate_AF,3) ) args->estimate_AF += 3;
else if ( !strncmp("PL,",args->estimate_AF,3) ) { args->estimate_AF += 3; args->af_from_PL = 1; }
if ( strcmp("-",args->estimate_AF) )
- args->af_smpl = smpl_ilist_init(args->hdr, args->estimate_AF, 1, SMPL_NONE);
+ args->af_smpl = smpl_ilist_init(args->hdr, args->estimate_AF, 1, SMPL_NONE|SMPL_VERBOSE);
}
if ( args->estimate_AF || args->fake_PLs )
error("Error: The FORMAT/GT tag not found in the header\n");
}
- args->roh_smpl = smpl_ilist_init(args->hdr, args->samples, args->samples_is_file, SMPL_NONE);
+ args->roh_smpl = smpl_ilist_init(args->hdr, args->samples, args->samples_is_file, SMPL_NONE|SMPL_VERBOSE);
if ( args->samples )
{
// we may be able to subset to a few samples, for a text VCF this can be a major speedup
if ( p[irr]<0 || p[ira]<0 || p[iaa]<0 ) continue; /* missing value */ \
if ( p[irr]==p[ira] && p[irr]==p[iaa] ) continue; /* all values are the same */ \
double prob[3], norm = 0; \
- prob[0] = p[irr] < 256 ? args->pl2p[ p[irr] ] : args->pl2p[255]; \
- prob[1] = p[ira] < 256 ? args->pl2p[ p[ira] ] : args->pl2p[255]; \
- prob[2] = p[iaa] < 256 ? args->pl2p[ p[iaa] ] : args->pl2p[255]; \
+ prob[0] = args->pl2p[ max255(p[irr]) ]; \
+ prob[1] = args->pl2p[ max255(p[ira]) ]; \
+ prob[2] = args->pl2p[ max255(p[iaa]) ]; \
for (j=0; j<3; j++) norm += prob[j]; \
for (j=0; j<3; j++) prob[j] /= norm; \
af += 0.5*prob[1] + prob[2]; \
if ( p[irr]<0 || p[ira]<0 || p[iaa]<0 ) continue; /* missing value */ \
if ( p[irr]==p[ira] && p[irr]==p[iaa] ) continue; /* all values are the same */ \
double prob[3], norm = 0; \
- prob[0] = p[irr] < 256 ? args->pl2p[ p[irr] ] : args->pl2p[255]; \
- prob[1] = p[ira] < 256 ? args->pl2p[ p[ira] ] : args->pl2p[255]; \
- prob[2] = p[iaa] < 256 ? args->pl2p[ p[iaa] ] : args->pl2p[255]; \
+ prob[0] = args->pl2p[ max255(p[irr]) ]; \
+ prob[1] = args->pl2p[ max255(p[ira]) ]; \
+ prob[2] = args->pl2p[ max255(p[iaa]) ]; \
for (j=0; j<3; j++) norm += prob[j]; \
for (j=0; j<3; j++) prob[j] /= norm; \
af += 0.5*prob[1] + prob[2]; \
if ( ret>0 )
alt_freq = args->AFs[ial-1];
if ( ret==-2 )
- error("Type mismatch for INFO/%s tag at %s:%d\n", args->af_tag, bcf_seqname(args->hdr,line), line->pos+1);
+ error("Type mismatch for INFO/%s tag at %s:%"PRId64"\n", args->af_tag, bcf_seqname(args->hdr,line), (int64_t) line->pos+1);
}
else if ( args->af_fname )
{
type_t *p = (type_t*)fmt_pl->p + fmt_pl->n*ismpl; \
if ( p[irr]<0 || p[ira]<0 || p[iaa]<0 ) continue; /* missing value */ \
if ( p[irr]==p[ira] && p[irr]==p[iaa] ) continue; /* all values are the same */ \
- pdg[0] = p[irr] < 256 ? args->pl2p[ p[irr] ] : args->pl2p[255]; \
- pdg[1] = p[ira] < 256 ? args->pl2p[ p[ira] ] : args->pl2p[255]; \
- pdg[2] = p[iaa] < 256 ? args->pl2p[ p[iaa] ] : args->pl2p[255]; \
+ pdg[0] = args->pl2p[ max255(p[irr]) ]; \
+ pdg[1] = args->pl2p[ max255(p[ira]) ]; \
+ pdg[2] = args->pl2p[ max255(p[iaa]) ]; \
}
switch (fmt_pl->type) {
case BCF_BT_INT8: BRANCH(int8_t); break;
fprintf(stderr, " -S, --samples-file <file> file of samples to analyze [all samples]\n");
fprintf(stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
- fprintf(stderr, " --threads <int> number of extra decompression threads [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(stderr, "\n");
fprintf(stderr, "HMM Options:\n");
fprintf(stderr, " -a, --hw-to-az <float> P(AZ|HW) transition probability from HW (Hardy-Weinberg) to AZ (autozygous) state [6.7e-8]\n");
}
}
if ( !args->output_fname ) args->output_fname = "stdout";
- if ( !args->output_type ) args->output_type = OUTPUT_ST|OUTPUT_RG;
+ if ( !args->output_type || args->output_type==OUTPUT_GZ ) args->output_type |= OUTPUT_ST|OUTPUT_RG;
char *fname = NULL;
if ( optind==argc )
{
}
if ( args->n_threads && bcf_sr_set_threads(args->files, args->n_threads)<0)
error("Failed to create threads\n");
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
init_data(args);
while ( bcf_sr_next_line(args->files) )
return mem;
}
+static inline int max255(int i)
+{
+ return i < 256 ? i : 255;
+}
+
static void init_data(args_t *args)
{
int i;
if ( !strncmp("GT,",args->estimate_AF,3) ) args->estimate_AF += 3;
else if ( !strncmp("PL,",args->estimate_AF,3) ) { args->estimate_AF += 3; args->af_from_PL = 1; }
if ( strcmp("-",args->estimate_AF) )
- args->af_smpl = smpl_ilist_init(args->hdr, args->estimate_AF, 1, SMPL_NONE);
+ args->af_smpl = smpl_ilist_init(args->hdr, args->estimate_AF, 1, SMPL_NONE|SMPL_VERBOSE);
}
if ( args->estimate_AF || args->fake_PLs )
error("Error: The FORMAT/GT tag not found in the header\n");
}
- args->roh_smpl = smpl_ilist_init(args->hdr, args->samples, args->samples_is_file, SMPL_NONE);
+ args->roh_smpl = smpl_ilist_init(args->hdr, args->samples, args->samples_is_file, SMPL_NONE|SMPL_VERBOSE);
if ( args->samples )
{
// we may be able to subset to a few samples, for a text VCF this can be a major speedup
if ( p[irr]<0 || p[ira]<0 || p[iaa]<0 ) continue; /* missing value */ \
if ( p[irr]==p[ira] && p[irr]==p[iaa] ) continue; /* all values are the same */ \
double prob[3], norm = 0; \
- prob[0] = p[irr] < 256 ? args->pl2p[ p[irr] ] : args->pl2p[255]; \
- prob[1] = p[ira] < 256 ? args->pl2p[ p[ira] ] : args->pl2p[255]; \
- prob[2] = p[iaa] < 256 ? args->pl2p[ p[iaa] ] : args->pl2p[255]; \
+ prob[0] = args->pl2p[ max255(p[irr]) ]; \
+ prob[1] = args->pl2p[ max255(p[ira]) ]; \
+ prob[2] = args->pl2p[ max255(p[iaa]) ]; \
for (j=0; j<3; j++) norm += prob[j]; \
for (j=0; j<3; j++) prob[j] /= norm; \
af += 0.5*prob[1] + prob[2]; \
if ( p[irr]<0 || p[ira]<0 || p[iaa]<0 ) continue; /* missing value */ \
if ( p[irr]==p[ira] && p[irr]==p[iaa] ) continue; /* all values are the same */ \
double prob[3], norm = 0; \
- prob[0] = p[irr] < 256 ? args->pl2p[ p[irr] ] : args->pl2p[255]; \
- prob[1] = p[ira] < 256 ? args->pl2p[ p[ira] ] : args->pl2p[255]; \
- prob[2] = p[iaa] < 256 ? args->pl2p[ p[iaa] ] : args->pl2p[255]; \
+ prob[0] = args->pl2p[ max255(p[irr]) ]; \
+ prob[1] = args->pl2p[ max255(p[ira]) ]; \
+ prob[2] = args->pl2p[ max255(p[iaa]) ]; \
for (j=0; j<3; j++) norm += prob[j]; \
for (j=0; j<3; j++) prob[j] /= norm; \
af += 0.5*prob[1] + prob[2]; \
if ( ret>0 )
alt_freq = args->AFs[ial-1];
if ( ret==-2 )
- error("Type mismatch for INFO/%s tag at %s:%d\n", args->af_tag, bcf_seqname(args->hdr,line), line->pos+1);
+ error("Type mismatch for INFO/%s tag at %s:%"PRId64"\n", args->af_tag, bcf_seqname(args->hdr,line), (int64_t) line->pos+1);
}
else if ( args->af_fname )
{
type_t *p = (type_t*)fmt_pl->p + fmt_pl->n*ismpl; \
if ( p[irr]<0 || p[ira]<0 || p[iaa]<0 ) continue; /* missing value */ \
if ( p[irr]==p[ira] && p[irr]==p[iaa] ) continue; /* all values are the same */ \
- pdg[0] = p[irr] < 256 ? args->pl2p[ p[irr] ] : args->pl2p[255]; \
- pdg[1] = p[ira] < 256 ? args->pl2p[ p[ira] ] : args->pl2p[255]; \
- pdg[2] = p[iaa] < 256 ? args->pl2p[ p[iaa] ] : args->pl2p[255]; \
+ pdg[0] = args->pl2p[ max255(p[irr]) ]; \
+ pdg[1] = args->pl2p[ max255(p[ira]) ]; \
+ pdg[2] = args->pl2p[ max255(p[iaa]) ]; \
}
switch (fmt_pl->type) {
case BCF_BT_INT8: BRANCH(int8_t); break;
fprintf(bcftools_stderr, " -S, --samples-file <file> file of samples to analyze [all samples]\n");
fprintf(bcftools_stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(bcftools_stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra decompression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(bcftools_stderr, "\n");
fprintf(bcftools_stderr, "HMM Options:\n");
fprintf(bcftools_stderr, " -a, --hw-to-az <float> P(AZ|HW) transition probability from HW (Hardy-Weinberg) to AZ (autozygous) state [6.7e-8]\n");
}
}
if ( !args->output_fname ) args->output_fname = "bcftools_stdout";
- if ( !args->output_type ) args->output_type = OUTPUT_ST|OUTPUT_RG;
+ if ( !args->output_type || args->output_type==OUTPUT_GZ ) args->output_type |= OUTPUT_ST|OUTPUT_RG;
char *fname = NULL;
if ( optind==argc )
{
}
if ( args->n_threads && bcf_sr_set_threads(args->files, args->n_threads)<0)
error("Failed to create threads\n");
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
init_data(args);
while ( bcf_sr_next_line(args->files) )
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/vcfutils.h>
+#include <htslib/hts_os.h>
#include <inttypes.h>
#include "bcftools.h"
if ( !som->w ) error("Could not alloc %"PRIu64" bytes [nbin=%d ndim=%d]\n", (uint64_t)(sizeof(double)*som->size),som->nbin,som->ndim);
int i;
for (i=0; i<som->size*som->kdim; i++)
- som->w[i] = (double)random()/RAND_MAX;
+ som->w[i] = random();
som->a_idx = (int*) malloc(sizeof(int)*som->ndim);
som->b_idx = (int*) malloc(sizeof(int)*som->ndim);
som->div = (double*) malloc(sizeof(double)*som->ndim);
case 't': args->action = SOM_TRAIN; break;
case 'c': args->action = SOM_CLASSIFY; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/vcfutils.h>
+#include <htslib/hts_os.h>
#include <inttypes.h>
#include "bcftools.h"
if ( !som->w ) error("Could not alloc %"PRIu64" bytes [nbin=%d ndim=%d]\n", (uint64_t)(sizeof(double)*som->size),som->nbin,som->ndim);
int i;
for (i=0; i<som->size*som->kdim; i++)
- som->w[i] = (double)random()/RAND_MAX;
+ som->w[i] = random();
som->a_idx = (int*) malloc(sizeof(int)*som->ndim);
som->b_idx = (int*) malloc(sizeof(int)*som->ndim);
som->div = (double*) malloc(sizeof(double)*som->ndim);
case 't': args->action = SOM_TRAIN; break;
case 'c': args->action = SOM_CLASSIFY; break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
#include <getopt.h>
#include <ctype.h>
#include <string.h>
+#include <strings.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <math.h>
+#ifdef _WIN32
+#include <windows.h>
+#endif
#include <htslib/vcf.h>
#include <htslib/kstring.h>
+#include <htslib/hts_os.h>
#include "kheap.h"
#include "bcftools.h"
}
args_t;
+void clean_files(args_t *args)
+{
+ int i;
+ fprintf(stderr,"Cleaning\n");
+ for (i=0; i<args->nblk; i++)
+ {
+ blk_t *blk = args->blk + i;
+ if ( blk->fname )
+ {
+ unlink(blk->fname);
+ free(blk->fname);
+ }
+ if ( blk->rec )
+ bcf_destroy(blk->rec);
+ }
+ rmdir(args->tmp_dir);
+}
+void clean_files_and_throw(args_t *args, const char *format, ...)
+{
+ va_list ap;
+ va_start(ap, format);
+ vfprintf(stderr, format, ap);
+ va_end(ap);
+ clean_files(args);
+ exit(-1);
+}
+
int cmp_bcf_pos(const void *aptr, const void *bptr)
{
bcf1_t *a = *((bcf1_t**)aptr);
kstring_t str = {0,0,0};
ksprintf(&str, "%s/%05d.bcf", args->tmp_dir, (int)args->nblk);
blk->fname = str.s;
+ blk->rec = NULL;
+ blk->fh = NULL;
htsFile *fh = hts_open(blk->fname, "wbu");
- if ( fh == NULL ) error("Cannot write %s: %s\n", blk->fname, strerror(errno));
- bcf_hdr_write(fh, args->hdr);
+ if ( fh == NULL ) clean_files_and_throw(args, "Cannot write %s: %s\n", blk->fname, strerror(errno));
+ if ( bcf_hdr_write(fh, args->hdr)!=0 ) clean_files_and_throw(args, "[%s] Error: cannot write to %s\n", __func__,blk->fname);
int i;
for (i=0; i<args->nbuf; i++)
{
- bcf_write(fh, args->hdr, args->buf[i]);
+ if ( bcf_write(fh, args->hdr, args->buf[i])!=0 ) clean_files_and_throw(args, "[%s] Error: cannot write to %s\n", __func__,blk->fname);
bcf_destroy(args->buf[i]);
}
- hts_close(fh);
+ if ( hts_close(fh)!=0 ) clean_files_and_throw(args, "[%s] Error: close failed .. %s\n", __func__,blk->fname);
args->nbuf = 0;
args->mem = 0;
void sort_blocks(args_t *args)
{
htsFile *in = hts_open(args->fname, "r");
- if ( !in ) error("Could not read %s\n", args->fname);
+ if ( !in ) clean_files_and_throw(args, "Could not read %s\n", args->fname);
args->hdr = bcf_hdr_read(in);
+ if ( !args->hdr) clean_files_and_throw(args, "Could not read VCF/BCF headers from %s\n", args->fname);
while ( 1 )
{
bcf1_t *rec = bcf_init();
int ret = bcf_read1(in, args->hdr, rec);
- if ( ret < -1 ) error("Error encountered while parsing the input\n");
+ if ( ret < -1 ) clean_files_and_throw(args,"Error encountered while parsing the input\n");
if ( ret == -1 )
{
bcf_destroy(rec);
break;
}
+ if ( rec->errcode ) clean_files_and_throw(args,"Error encountered while parsing the input at %s:%d\n",bcf_seqname(args->hdr,rec),rec->pos+1);
buf_push(args, rec);
}
buf_flush(args);
free(args->buf);
- if ( hts_close(in)!=0 ) error("Close failed: %s\n", args->fname);
+ if ( hts_close(in)!=0 ) clean_files_and_throw(args,"Close failed: %s\n", args->fname);
}
static inline int blk_is_smaller(blk_t **aptr, blk_t **bptr)
}
KHEAP_INIT(blk, blk_t*, blk_is_smaller)
-void blk_read(khp_blk_t *bhp, bcf_hdr_t *hdr, blk_t *blk)
+void blk_read(args_t *args, khp_blk_t *bhp, bcf_hdr_t *hdr, blk_t *blk)
{
if ( !blk->fh ) return;
int ret = bcf_read(blk->fh, hdr, blk->rec);
- if ( ret < -1 ) error("Error reading %s\n", blk->fname);
+ if ( ret < -1 ) clean_files_and_throw(args, "Error reading %s\n", blk->fname);
if ( ret == -1 )
{
- if ( hts_close(blk->fh)!=0 ) error("Close failed: %s\n", blk->fname);
+ if ( hts_close(blk->fh)!=0 ) clean_files_and_throw(args, "Close failed: %s\n", blk->fname);
blk->fh = 0;
return;
}
{
blk_t *blk = args->blk + i;
blk->fh = hts_open(blk->fname, "r");
- if ( !blk->fh ) error("Could not read %s: %s\n", blk->fname, strerror(errno));
+ if ( !blk->fh ) clean_files_and_throw(args, "Could not read %s: %s\n", blk->fname, strerror(errno));
bcf_hdr_t *hdr = bcf_hdr_read(blk->fh);
bcf_hdr_destroy(hdr);
blk->rec = bcf_init();
- blk_read(bhp, args->hdr, blk);
+ blk_read(args, bhp, args->hdr, blk);
}
htsFile *out = hts_open(args->output_fname, hts_bcf_wmode(args->output_type));
- bcf_hdr_write(out, args->hdr);
+ if ( bcf_hdr_write(out, args->hdr)!=0 ) clean_files_and_throw(args, "[%s] Error: cannot write to %s\n", __func__,args->output_fname);
while ( bhp->ndat )
{
blk_t *blk = bhp->dat[0];
- bcf_write(out, args->hdr, blk->rec);
+ if ( bcf_write(out, args->hdr, blk->rec)!=0 ) clean_files_and_throw(args, "[%s] Error: cannot write to %s\n", __func__,args->output_fname);
khp_delete(blk, bhp);
- blk_read(bhp, args->hdr, blk);
+ blk_read(args, bhp, args->hdr, blk);
}
- if ( hts_close(out)!=0 ) error("Close failed: %s\n", args->output_fname);
+ if ( hts_close(out)!=0 ) clean_files_and_throw(args, "Close failed: %s\n", args->output_fname);
+
+ clean_files(args);
- fprintf(stderr,"Cleaning\n");
- for (i=0; i<args->nblk; i++)
- {
- blk_t *blk = args->blk + i;
- unlink(blk->fname);
- free(blk->fname);
- bcf_destroy(blk->rec);
- }
- rmdir(args->tmp_dir);
free(args->blk);
khp_destroy(blk, bhp);
fprintf(stderr,"Done\n");
fprintf(stderr, " -m, --max-mem <float>[kMG] maximum memory to use [768M]\n"); // using metric units, 1M=1e6
fprintf(stderr, " -o, --output-file <file> output file name [stdout]\n");
fprintf(stderr, " -O, --output-type <b|u|z|v> b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n");
- fprintf(stderr, " -T, --temp-dir <dir> temporary files [/tmp/bcftools-sort.XXXXXX/]\n");
+ fprintf(stderr, " -T, --temp-dir <dir> temporary files [/tmp/bcftools-sort.XXXXXX]\n");
fprintf(stderr, "\n");
exit(1);
}
}
void mkdir_p(const char *fmt, ...);
-void init(args_t *args)
+static void init(args_t *args)
{
- if ( !args->tmp_dir )
+#ifdef _WIN32
+ char tmp_path[MAX_PATH];
+ int ret = GetTempPath(MAX_PATH, tmp_path);
+ if (!ret || ret > MAX_PATH)
+ error("Could not get the path to the temporary folder\n");
+ if (strlen(tmp_path) + strlen("/bcftools-sort.XXXXXX") >= MAX_PATH)
+ error("Full path to the temporary folder is too long\n");
+ strcat(tmp_path, "/bcftools-sort.XXXXXX");
+ args->tmp_dir = strdup(tmp_path);
+#else
+ args->tmp_dir = args->tmp_dir ? strdup(args->tmp_dir) : strdup("/tmp/bcftools-sort.XXXXXX");
+#endif
+ size_t len = strlen(args->tmp_dir);
+ if ( !strcmp("XXXXXX",args->tmp_dir+len-6) )
{
- args->tmp_dir = strdup("/tmp/bcftools-sort.XXXXXX");
- char *tmp_dir = mkdtemp(args->tmp_dir);
- if ( !tmp_dir ) error("mkdtemp(%s) failed: %s\n", args->tmp_dir,strerror(errno));
+#ifdef _WIN32
+ int ret = mkdir(mktemp(args->tmp_dir), 0700);
+ if ( ret ) error("mkdir(%s) failed: %s\n", args->tmp_dir,strerror(errno));
+#else
+ char *tmp = mkdtemp(args->tmp_dir);
+ if ( !tmp ) error("mkdtemp(%s) failed: %s\n", args->tmp_dir,strerror(errno));
+ int ret = chmod(tmp, S_IRUSR|S_IWUSR|S_IXUSR);
+ if ( ret ) error("chmod(%s,S_IRUSR|S_IWUSR|S_IXUSR) failed: %s\n", args->tmp_dir,strerror(errno));
+#endif
}
- else
- {
- args->tmp_dir = strdup(args->tmp_dir);
- mkdir_p(args->tmp_dir);
+ else {
+ mkdir_p("%s/",args->tmp_dir);
}
+
fprintf(stderr,"Writing to %s\n", args->tmp_dir);
}
-void destroy(args_t *args)
+static void destroy(args_t *args)
{
bcf_hdr_destroy(args->hdr);
free(args->tmp_dir);
default: error("The output type \"%s\" not recognised\n", optarg);
};
break;
- case 'h': usage(args);
- case '?': usage(args);
+ case 'h':
+ case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
}
#include <getopt.h>
#include <ctype.h>
#include <string.h>
+#include <strings.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <math.h>
+#ifdef _WIN32
+#include <windows.h>
+#endif
#include <htslib/vcf.h>
#include <htslib/kstring.h>
+#include <htslib/hts_os.h>
#include "kheap.h"
#include "bcftools.h"
}
args_t;
+void clean_files(args_t *args)
+{
+ int i;
+ fprintf(bcftools_stderr,"Cleaning\n");
+ for (i=0; i<args->nblk; i++)
+ {
+ blk_t *blk = args->blk + i;
+ if ( blk->fname )
+ {
+ unlink(blk->fname);
+ free(blk->fname);
+ }
+ if ( blk->rec )
+ bcf_destroy(blk->rec);
+ }
+ rmdir(args->tmp_dir);
+}
+void clean_files_and_throw(args_t *args, const char *format, ...)
+{
+ va_list ap;
+ va_start(ap, format);
+ vfprintf(bcftools_stderr, format, ap);
+ va_end(ap);
+ clean_files(args);
+ exit(-1);
+}
+
int cmp_bcf_pos(const void *aptr, const void *bptr)
{
bcf1_t *a = *((bcf1_t**)aptr);
kstring_t str = {0,0,0};
ksprintf(&str, "%s/%05d.bcf", args->tmp_dir, (int)args->nblk);
blk->fname = str.s;
+ blk->rec = NULL;
+ blk->fh = NULL;
htsFile *fh = hts_open(blk->fname, "wbu");
- if ( fh == NULL ) error("Cannot write %s: %s\n", blk->fname, strerror(errno));
- bcf_hdr_write(fh, args->hdr);
+ if ( fh == NULL ) clean_files_and_throw(args, "Cannot write %s: %s\n", blk->fname, strerror(errno));
+ if ( bcf_hdr_write(fh, args->hdr)!=0 ) clean_files_and_throw(args, "[%s] Error: cannot write to %s\n", __func__,blk->fname);
int i;
for (i=0; i<args->nbuf; i++)
{
- bcf_write(fh, args->hdr, args->buf[i]);
+ if ( bcf_write(fh, args->hdr, args->buf[i])!=0 ) clean_files_and_throw(args, "[%s] Error: cannot write to %s\n", __func__,blk->fname);
bcf_destroy(args->buf[i]);
}
- hts_close(fh);
+ if ( hts_close(fh)!=0 ) clean_files_and_throw(args, "[%s] Error: close failed .. %s\n", __func__,blk->fname);
args->nbuf = 0;
args->mem = 0;
void sort_blocks(args_t *args)
{
htsFile *in = hts_open(args->fname, "r");
- if ( !in ) error("Could not read %s\n", args->fname);
+ if ( !in ) clean_files_and_throw(args, "Could not read %s\n", args->fname);
args->hdr = bcf_hdr_read(in);
+ if ( !args->hdr) clean_files_and_throw(args, "Could not read VCF/BCF headers from %s\n", args->fname);
while ( 1 )
{
bcf1_t *rec = bcf_init();
int ret = bcf_read1(in, args->hdr, rec);
- if ( ret < -1 ) error("Error encountered while parsing the input\n");
+ if ( ret < -1 ) clean_files_and_throw(args,"Error encountered while parsing the input\n");
if ( ret == -1 )
{
bcf_destroy(rec);
break;
}
+ if ( rec->errcode ) clean_files_and_throw(args,"Error encountered while parsing the input at %s:%d\n",bcf_seqname(args->hdr,rec),rec->pos+1);
buf_push(args, rec);
}
buf_flush(args);
free(args->buf);
- if ( hts_close(in)!=0 ) error("Close failed: %s\n", args->fname);
+ if ( hts_close(in)!=0 ) clean_files_and_throw(args,"Close failed: %s\n", args->fname);
}
static inline int blk_is_smaller(blk_t **aptr, blk_t **bptr)
}
KHEAP_INIT(blk, blk_t*, blk_is_smaller)
-void blk_read(khp_blk_t *bhp, bcf_hdr_t *hdr, blk_t *blk)
+void blk_read(args_t *args, khp_blk_t *bhp, bcf_hdr_t *hdr, blk_t *blk)
{
if ( !blk->fh ) return;
int ret = bcf_read(blk->fh, hdr, blk->rec);
- if ( ret < -1 ) error("Error reading %s\n", blk->fname);
+ if ( ret < -1 ) clean_files_and_throw(args, "Error reading %s\n", blk->fname);
if ( ret == -1 )
{
- if ( hts_close(blk->fh)!=0 ) error("Close failed: %s\n", blk->fname);
+ if ( hts_close(blk->fh)!=0 ) clean_files_and_throw(args, "Close failed: %s\n", blk->fname);
blk->fh = 0;
return;
}
{
blk_t *blk = args->blk + i;
blk->fh = hts_open(blk->fname, "r");
- if ( !blk->fh ) error("Could not read %s: %s\n", blk->fname, strerror(errno));
+ if ( !blk->fh ) clean_files_and_throw(args, "Could not read %s: %s\n", blk->fname, strerror(errno));
bcf_hdr_t *hdr = bcf_hdr_read(blk->fh);
bcf_hdr_destroy(hdr);
blk->rec = bcf_init();
- blk_read(bhp, args->hdr, blk);
+ blk_read(args, bhp, args->hdr, blk);
}
htsFile *out = hts_open(args->output_fname, hts_bcf_wmode(args->output_type));
- bcf_hdr_write(out, args->hdr);
+ if ( bcf_hdr_write(out, args->hdr)!=0 ) clean_files_and_throw(args, "[%s] Error: cannot write to %s\n", __func__,args->output_fname);
while ( bhp->ndat )
{
blk_t *blk = bhp->dat[0];
- bcf_write(out, args->hdr, blk->rec);
+ if ( bcf_write(out, args->hdr, blk->rec)!=0 ) clean_files_and_throw(args, "[%s] Error: cannot write to %s\n", __func__,args->output_fname);
khp_delete(blk, bhp);
- blk_read(bhp, args->hdr, blk);
+ blk_read(args, bhp, args->hdr, blk);
}
- if ( hts_close(out)!=0 ) error("Close failed: %s\n", args->output_fname);
+ if ( hts_close(out)!=0 ) clean_files_and_throw(args, "Close failed: %s\n", args->output_fname);
+
+ clean_files(args);
- fprintf(bcftools_stderr,"Cleaning\n");
- for (i=0; i<args->nblk; i++)
- {
- blk_t *blk = args->blk + i;
- unlink(blk->fname);
- free(blk->fname);
- bcf_destroy(blk->rec);
- }
- rmdir(args->tmp_dir);
free(args->blk);
khp_destroy(blk, bhp);
fprintf(bcftools_stderr,"Done\n");
fprintf(bcftools_stderr, " -m, --max-mem <float>[kMG] maximum memory to use [768M]\n"); // using metric units, 1M=1e6
fprintf(bcftools_stderr, " -o, --output-file <file> output file name [bcftools_stdout]\n");
fprintf(bcftools_stderr, " -O, --output-type <b|u|z|v> b: compressed BCF, u: uncompressed BCF, z: compressed VCF, v: uncompressed VCF [v]\n");
- fprintf(bcftools_stderr, " -T, --temp-dir <dir> temporary files [/tmp/bcftools-sort.XXXXXX/]\n");
+ fprintf(bcftools_stderr, " -T, --temp-dir <dir> temporary files [/tmp/bcftools-sort.XXXXXX]\n");
fprintf(bcftools_stderr, "\n");
exit(1);
}
}
void mkdir_p(const char *fmt, ...);
-void init(args_t *args)
+static void init(args_t *args)
{
- if ( !args->tmp_dir )
+#ifdef _WIN32
+ char tmp_path[MAX_PATH];
+ int ret = GetTempPath(MAX_PATH, tmp_path);
+ if (!ret || ret > MAX_PATH)
+ error("Could not get the path to the temporary folder\n");
+ if (strlen(tmp_path) + strlen("/bcftools-sort.XXXXXX") >= MAX_PATH)
+ error("Full path to the temporary folder is too long\n");
+ strcat(tmp_path, "/bcftools-sort.XXXXXX");
+ args->tmp_dir = strdup(tmp_path);
+#else
+ args->tmp_dir = args->tmp_dir ? strdup(args->tmp_dir) : strdup("/tmp/bcftools-sort.XXXXXX");
+#endif
+ size_t len = strlen(args->tmp_dir);
+ if ( !strcmp("XXXXXX",args->tmp_dir+len-6) )
{
- args->tmp_dir = strdup("/tmp/bcftools-sort.XXXXXX");
- char *tmp_dir = mkdtemp(args->tmp_dir);
- if ( !tmp_dir ) error("mkdtemp(%s) failed: %s\n", args->tmp_dir,strerror(errno));
+#ifdef _WIN32
+ int ret = mkdir(mktemp(args->tmp_dir), 0700);
+ if ( ret ) error("mkdir(%s) failed: %s\n", args->tmp_dir,strerror(errno));
+#else
+ char *tmp = mkdtemp(args->tmp_dir);
+ if ( !tmp ) error("mkdtemp(%s) failed: %s\n", args->tmp_dir,strerror(errno));
+ int ret = chmod(tmp, S_IRUSR|S_IWUSR|S_IXUSR);
+ if ( ret ) error("chmod(%s,S_IRUSR|S_IWUSR|S_IXUSR) failed: %s\n", args->tmp_dir,strerror(errno));
+#endif
}
- else
- {
- args->tmp_dir = strdup(args->tmp_dir);
- mkdir_p(args->tmp_dir);
+ else {
+ mkdir_p("%s/",args->tmp_dir);
}
+
fprintf(bcftools_stderr,"Writing to %s\n", args->tmp_dir);
}
-void destroy(args_t *args)
+static void destroy(args_t *args)
{
bcf_hdr_destroy(args->hdr);
free(args->tmp_dir);
default: error("The output type \"%s\" not recognised\n", optarg);
};
break;
- case 'h': usage(args);
- case '?': usage(args);
+ case 'h':
+ case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
}
typedef struct
{
- int n_snps, n_indels, n_mnps, n_others, n_mals, n_snp_mals, n_records, n_noalts;
+ uint32_t n_snps, n_indels, n_mnps, n_others, n_mals, n_snp_mals, n_records, n_noalts;
int *af_ts, *af_tv, *af_snps; // first bin of af_* stats are singletons
#if HWE_STATS
int *af_hwe;
int subst[15];
int *smpl_hets, *smpl_homRR, *smpl_homAA, *smpl_ts, *smpl_tv, *smpl_indels, *smpl_ndp, *smpl_sngl;
int *smpl_hapRef, *smpl_hapAlt, *smpl_missing;
- int *smpl_indel_hets, *smpl_indel_homs;
+ int *smpl_ins_hets, *smpl_del_hets, *smpl_ins_homs, *smpl_del_homs;
int *smpl_frm_shifts; // not-applicable, in-frame, out-frame
unsigned long int *smpl_dp;
idist_t dp, dp_sites;
int nusr;
user_stats_t *usr;
+ double *dvaf; // distribution of the mean indel-allele frequency by length: -m_indel,-(m_indel-1),...-1,0,1,..,m_indel
+ uint32_t *nvaf;
}
stats_t;
stats->smpl_homRR = (int *) calloc(args->files->n_smpl,sizeof(int));
stats->smpl_hapRef = (int *) calloc(args->files->n_smpl,sizeof(int));
stats->smpl_hapAlt = (int *) calloc(args->files->n_smpl,sizeof(int));
- stats->smpl_indel_hets = (int *) calloc(args->files->n_smpl,sizeof(int));
- stats->smpl_indel_homs = (int *) calloc(args->files->n_smpl,sizeof(int));
+ stats->smpl_ins_hets = (int *) calloc(args->files->n_smpl,sizeof(int));
+ stats->smpl_del_hets = (int *) calloc(args->files->n_smpl,sizeof(int));
+ stats->smpl_ins_homs = (int *) calloc(args->files->n_smpl,sizeof(int));
+ stats->smpl_del_homs = (int *) calloc(args->files->n_smpl,sizeof(int));
stats->smpl_ts = (int *) calloc(args->files->n_smpl,sizeof(int));
stats->smpl_tv = (int *) calloc(args->files->n_smpl,sizeof(int));
stats->smpl_indels = (int *) calloc(args->files->n_smpl,sizeof(int));
#endif
if ( args->exons_fname )
stats->smpl_frm_shifts = (int*) calloc(args->files->n_smpl*3,sizeof(int));
+ stats->nvaf = (uint32_t*) calloc(stats->m_indel*2+1,sizeof(*stats->nvaf));
+ stats->dvaf = (double*) calloc(stats->m_indel*2+1,sizeof(*stats->dvaf));
}
idist_init(&stats->dp, args->dp_min,args->dp_max,args->dp_step);
idist_init(&stats->dp_sites, args->dp_min,args->dp_max,args->dp_step);
free(stats->smpl_homRR);
free(stats->smpl_hapRef);
free(stats->smpl_hapAlt);
- free(stats->smpl_indel_homs);
- free(stats->smpl_indel_hets);
+ free(stats->smpl_ins_homs);
+ free(stats->smpl_del_homs);
+ free(stats->smpl_ins_hets);
+ free(stats->smpl_del_hets);
free(stats->smpl_ts);
free(stats->smpl_tv);
free(stats->smpl_indels);
}
free(stats->usr);
if ( args->exons ) free(stats->smpl_frm_shifts);
+ free(stats->nvaf);
+ free(stats->dvaf);
}
for (j=0; j<args->nusr; j++) free(args->usr[j].tag);
if ( args->af_bins ) bin_destroy(args->af_bins);
}
}
+static inline void update_dvaf(stats_t *stats, bcf1_t *line, bcf_fmt_t *fmt, int ismpl, int ial, int jal)
+{
+ if ( !fmt ) return;
+
+ float dvaf;
+ #define BRANCH_INT(type_t,missing,vector_end) { \
+ type_t *p = (type_t *) (fmt->p + fmt->size*ismpl); \
+ if ( p[ial]==vector_end || p[jal]==vector_end ) return; \
+ if ( p[ial]==missing || p[jal]==missing ) return; \
+ if ( !p[ial] && !p[jal] ) return; \
+ dvaf = (float)p[ial]/(p[ial]+p[jal]); \
+ }
+ switch (fmt->type) {
+ case BCF_BT_INT8: BRANCH_INT(int8_t, bcf_int8_missing, bcf_int8_vector_end); break;
+ case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_missing, bcf_int16_vector_end); break;
+ case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_missing, bcf_int32_vector_end); break;
+ default: fprintf(stderr, "[E::%s] todo: %d\n", __func__, fmt->type); exit(1); break;
+ }
+ #undef BRANCH_INT
+
+ int len = line->d.var[ial].n;
+ if ( len < -stats->m_indel ) len = -stats->m_indel;
+ else if ( len > stats->m_indel ) len = stats->m_indel;
+ int bin = stats->m_indel + len;
+ stats->nvaf[bin]++;
+ stats->dvaf[bin] += dvaf;
+}
+
static void do_sample_stats(args_t *args, stats_t *stats, bcf_sr_t *reader, int matched)
{
bcf_srs_t *files = args->files;
if ( (fmt_ptr = bcf_get_fmt(reader->header,reader->buffer[0],"GT")) )
{
+ bcf_fmt_t *ad_fmt_ptr = bcf_get_variant_types(line)&VCF_INDEL ? bcf_get_fmt(reader->header,reader->buffer[0],"AD") : NULL;
+
int ref = bcf_acgt2int(*line->d.allele[0]);
int is, n_nref = 0, i_nref = 0;
for (is=0; is<args->files->n_smpl; is++)
if ( gt != GT_HOM_RR )
{
stats->smpl_indels[is]++;
- if ( gt==GT_HET_RA || gt==GT_HET_AA ) stats->smpl_indel_hets[is]++;
- else if ( gt==GT_HOM_AA ) stats->smpl_indel_homs[is]++;
+
+ if ( gt==GT_HET_RA || gt==GT_HET_AA )
+ {
+ int is_ins = 0, is_del = 0;
+ if ( bcf_get_variant_type(line,ial)&VCF_INDEL )
+ {
+ if ( line->d.var[ial].n < 0 ) is_del = 1;
+ else is_ins = 1;
+ update_dvaf(stats,line,ad_fmt_ptr,is,ial,jal);
+ }
+ if ( bcf_get_variant_type(line,jal)&VCF_INDEL )
+ {
+ if ( line->d.var[jal].n < 0 ) is_del = 1;
+ else is_ins = 1;
+ update_dvaf(stats,line,ad_fmt_ptr,is,jal,ial);
+ }
+ // Note that alt-het genotypes with both ins and del allele are counted twice!!
+ if ( is_del ) stats->smpl_del_hets[is]++;
+ if ( is_ins ) stats->smpl_ins_hets[is]++;
+ }
+ else if ( gt==GT_HOM_AA )
+ {
+ if ( line->d.var[ial].n < 0 ) stats->smpl_del_homs[is]++;
+ else stats->smpl_ins_homs[is]++;
+ }
}
if ( stats->smpl_frm_shifts )
{
}
#undef BRANCH_INT
}
+ else if ( (fmt_ptr = bcf_get_fmt(reader->header,reader->buffer[0],"AD")) )
+ {
+ #define BRANCH_INT(type_t,missing,vector_end) { \
+ int is,iv; \
+ for (is=0; is<args->files->n_smpl; is++) \
+ { \
+ type_t *p = (type_t *) (fmt_ptr->p + fmt_ptr->size*is); \
+ int dp = 0, has_value = 0; \
+ for (iv=0; iv<fmt_ptr->n; iv++) \
+ { \
+ if ( p[iv]==vector_end ) break; \
+ if ( p[iv]==missing ) continue; \
+ has_value = 1; \
+ dp += p[iv]; \
+ } \
+ if ( has_value ) \
+ { \
+ (*idist(&stats->dp, dp))++; \
+ stats->smpl_ndp[is]++; \
+ stats->smpl_dp[is] += dp; \
+ } \
+ } \
+ }
+ switch (fmt_ptr->type) {
+ case BCF_BT_INT8: BRANCH_INT(int8_t, bcf_int8_missing, bcf_int8_vector_end); break;
+ case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_missing, bcf_int16_vector_end); break;
+ case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_missing, bcf_int32_vector_end); break;
+ default: fprintf(stderr, "[E::%s] todo: %d\n", __func__, fmt_ptr->type); exit(1); break;
+ }
+ #undef BRANCH_INT
+ }
if ( matched==3 )
{
fmt1 = bcf_get_fmt(files->readers[1].header,files->readers[1].buffer[0],"GT"); if ( !fmt1 ) return;
// only the first ALT allele is considered
+ if (args->ntmp_iaf <= 1) return; // Do not consider invariate sites
int iaf = args->tmp_iaf[1];
int line_type = bcf_get_variant_types(files->readers[0].buffer[0]);
gtcmp_t *af_stats = line_type&VCF_SNP ? args->af_gts_snps : args->af_gts_indels;
{
nmm++;
bcf_sr_t *reader = &files->readers[0];
- printf("DBG\t%s\t%d\t%s\t%d\t%d\n",reader->header->id[BCF_DT_CTG][reader->buffer[0]->rid].key,reader->buffer[0]->pos+1,files->samples[is],gt,gt2);
+ printf("DBG\t%s\t%"PRId64"\t%s\t%d\t%d\n",reader->header->id[BCF_DT_CTG][reader->buffer[0]->rid].key,(int64_t) reader->buffer[0]->pos+1,files->samples[is],gt,gt2);
}
else
{
}
}
float nrd = nrefm+nmm ? 100.*nmm/(nrefm+nmm) : 0;
- printf("PSD\t%s\t%d\t%d\t%d\t%f\n", reader->header->id[BCF_DT_CTG][reader->buffer[0]->rid].key,reader->buffer[0]->pos+1,nm,nmm,nrd);
+ printf("PSD\t%s\t%"PRId64"\t%d\t%d\t%f\n", reader->header->id[BCF_DT_CTG][reader->buffer[0]->rid].key,(int64_t) reader->buffer[0]->pos+1,nm,nmm,nrd);
}
}
}
for (id=0; id<args->nstats; id++)
{
stats_t *stats = &args->stats[id];
- printf("SN\t%d\tnumber of records:\t%d\n", id, stats->n_records);
- printf("SN\t%d\tnumber of no-ALTs:\t%d\n", id, stats->n_noalts);
- printf("SN\t%d\tnumber of SNPs:\t%d\n", id, stats->n_snps);
- printf("SN\t%d\tnumber of MNPs:\t%d\n", id, stats->n_mnps);
- printf("SN\t%d\tnumber of indels:\t%d\n", id, stats->n_indels);
- printf("SN\t%d\tnumber of others:\t%d\n", id, stats->n_others);
- printf("SN\t%d\tnumber of multiallelic sites:\t%d\n", id, stats->n_mals);
- printf("SN\t%d\tnumber of multiallelic SNP sites:\t%d\n", id, stats->n_snp_mals);
+ printf("SN\t%d\tnumber of records:\t%u\n", id, stats->n_records);
+ printf("SN\t%d\tnumber of no-ALTs:\t%u\n", id, stats->n_noalts);
+ printf("SN\t%d\tnumber of SNPs:\t%u\n", id, stats->n_snps);
+ printf("SN\t%d\tnumber of MNPs:\t%u\n", id, stats->n_mnps);
+ printf("SN\t%d\tnumber of indels:\t%u\n", id, stats->n_indels);
+ printf("SN\t%d\tnumber of others:\t%u\n", id, stats->n_others);
+ printf("SN\t%d\tnumber of multiallelic sites:\t%u\n", id, stats->n_mals);
+ printf("SN\t%d\tnumber of multiallelic SNP sites:\t%u\n", id, stats->n_snp_mals);
}
printf("# TSTV, transitions/transversions:\n# TSTV\t[2]id\t[3]ts\t[4]tv\t[5]ts/tv\t[6]ts (1st ALT)\t[7]tv (1st ALT)\t[8]ts/tv (1st ALT)\n");
for (id=0; id<args->nstats; id++)
}
}
}
- printf("# IDD, InDel distribution:\n# IDD\t[2]id\t[3]length (deletions negative)\t[4]count\n");
+ printf("# IDD, InDel distribution:\n# IDD\t[2]id\t[3]length (deletions negative)\t[4]number of sites\t[5]number of genotypes\t[6]mean VAF\n");
for (id=0; id<args->nstats; id++)
{
stats_t *stats = &args->stats[id];
for (i=stats->m_indel-1; i>=0; i--)
- if ( stats->deletions[i] ) printf("IDD\t%d\t%d\t%d\n", id,-i-1,stats->deletions[i]);
+ {
+ if ( !stats->deletions[i] ) continue;
+ // whops, differently organized arrow, dels are together with ins
+ int bin = stats->m_indel - i - 1;
+ printf("IDD\t%d\t%d\t%d\t", id,-i-1,stats->deletions[i]);
+ if ( stats->nvaf && stats->nvaf[bin] )
+ printf("%u\t%.2f",stats->nvaf[bin],stats->dvaf[bin]/stats->nvaf[bin]);
+ else
+ printf("0\t.");
+ printf("\n");
+ }
for (i=0; i<stats->m_indel; i++)
- if ( stats->insertions[i] ) printf("IDD\t%d\t%d\t%d\n", id,i+1,stats->insertions[i]);
+ {
+ if ( !stats->insertions[i] ) continue;
+ int bin = stats->m_indel + i + 1;
+ printf("IDD\t%d\t%d\t%d\t", id,i+1,stats->insertions[i]);
+ if ( stats->nvaf && stats->nvaf[bin] )
+ printf("%u\t%.2f",stats->nvaf[bin],stats->dvaf[bin]/stats->nvaf[bin]);
+ else
+ printf("0\t.");
+ printf("\n");
+ }
}
printf("# ST, Substitution types:\n# ST\t[2]id\t[3]type\t[4]count\n");
for (id=0; id<args->nstats; id++)
}
}
-
- printf("# PSI, Per-Sample Indels\n# PSI\t[2]id\t[3]sample\t[4]in-frame\t[5]out-frame\t[6]not applicable\t[7]out/(in+out) ratio\t[8]nHets\t[9]nAA\n");
+ printf("# PSI, Per-Sample Indels. Note that alt-het genotypes with both ins and del allele are counted twice, in both nInsHets and nDelHets.\n");
+ printf("# PSI\t[2]id\t[3]sample\t[4]in-frame\t[5]out-frame\t[6]not applicable\t[7]out/(in+out) ratio\t[8]nInsHets\t[9]nDelHets\t[10]nInsAltHoms\t[11]nDelAltHoms\n");
for (id=0; id<args->nstats; id++)
{
stats_t *stats = &args->stats[id];
in = stats->smpl_frm_shifts[i*3 + 1];
out = stats->smpl_frm_shifts[i*3 + 2];
}
- int nhom = stats->smpl_indel_homs[i];
- int nhet = stats->smpl_indel_hets[i];
- printf("PSI\t%d\t%s\t%d\t%d\t%d\t%.2f\t%d\t%d\n", id,args->files->samples[i], in,out,na,in+out?1.0*out/(in+out):0,nhet,nhom);
+ printf("PSI\t%d\t%s\t%d\t%d\t%d\t%.2f\t%d\t%d\t%d\t%d\n", id,args->files->samples[i], in,out,na,in+out?1.0*out/(in+out):0,
+ stats->smpl_ins_hets[i],stats->smpl_del_hets[i],stats->smpl_ins_homs[i],stats->smpl_del_homs[i]);
}
}
fprintf(stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
fprintf(stderr, " -u, --user-tstv <TAG[:min:max:n]> collect Ts/Tv stats for any tag using the given binning [0:1:100]\n");
- fprintf(stderr, " --threads <int> number of extra decompression threads [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(stderr, " -v, --verbose produce verbose per-site and per-sample output\n");
fprintf(stderr, "\n");
exit(1);
case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
while (fname)
{
if ( !bcf_sr_add_reader(args->files, fname) )
- error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
fname = ++optind < argc ? argv[optind] : NULL;
}
typedef struct
{
- int n_snps, n_indels, n_mnps, n_others, n_mals, n_snp_mals, n_records, n_noalts;
+ uint32_t n_snps, n_indels, n_mnps, n_others, n_mals, n_snp_mals, n_records, n_noalts;
int *af_ts, *af_tv, *af_snps; // first bin of af_* stats are singletons
#if HWE_STATS
int *af_hwe;
int subst[15];
int *smpl_hets, *smpl_homRR, *smpl_homAA, *smpl_ts, *smpl_tv, *smpl_indels, *smpl_ndp, *smpl_sngl;
int *smpl_hapRef, *smpl_hapAlt, *smpl_missing;
- int *smpl_indel_hets, *smpl_indel_homs;
+ int *smpl_ins_hets, *smpl_del_hets, *smpl_ins_homs, *smpl_del_homs;
int *smpl_frm_shifts; // not-applicable, in-frame, out-frame
unsigned long int *smpl_dp;
idist_t dp, dp_sites;
int nusr;
user_stats_t *usr;
+ double *dvaf; // distribution of the mean indel-allele frequency by length: -m_indel,-(m_indel-1),...-1,0,1,..,m_indel
+ uint32_t *nvaf;
}
stats_t;
stats->smpl_homRR = (int *) calloc(args->files->n_smpl,sizeof(int));
stats->smpl_hapRef = (int *) calloc(args->files->n_smpl,sizeof(int));
stats->smpl_hapAlt = (int *) calloc(args->files->n_smpl,sizeof(int));
- stats->smpl_indel_hets = (int *) calloc(args->files->n_smpl,sizeof(int));
- stats->smpl_indel_homs = (int *) calloc(args->files->n_smpl,sizeof(int));
+ stats->smpl_ins_hets = (int *) calloc(args->files->n_smpl,sizeof(int));
+ stats->smpl_del_hets = (int *) calloc(args->files->n_smpl,sizeof(int));
+ stats->smpl_ins_homs = (int *) calloc(args->files->n_smpl,sizeof(int));
+ stats->smpl_del_homs = (int *) calloc(args->files->n_smpl,sizeof(int));
stats->smpl_ts = (int *) calloc(args->files->n_smpl,sizeof(int));
stats->smpl_tv = (int *) calloc(args->files->n_smpl,sizeof(int));
stats->smpl_indels = (int *) calloc(args->files->n_smpl,sizeof(int));
#endif
if ( args->exons_fname )
stats->smpl_frm_shifts = (int*) calloc(args->files->n_smpl*3,sizeof(int));
+ stats->nvaf = (uint32_t*) calloc(stats->m_indel*2+1,sizeof(*stats->nvaf));
+ stats->dvaf = (double*) calloc(stats->m_indel*2+1,sizeof(*stats->dvaf));
}
idist_init(&stats->dp, args->dp_min,args->dp_max,args->dp_step);
idist_init(&stats->dp_sites, args->dp_min,args->dp_max,args->dp_step);
free(stats->smpl_homRR);
free(stats->smpl_hapRef);
free(stats->smpl_hapAlt);
- free(stats->smpl_indel_homs);
- free(stats->smpl_indel_hets);
+ free(stats->smpl_ins_homs);
+ free(stats->smpl_del_homs);
+ free(stats->smpl_ins_hets);
+ free(stats->smpl_del_hets);
free(stats->smpl_ts);
free(stats->smpl_tv);
free(stats->smpl_indels);
}
free(stats->usr);
if ( args->exons ) free(stats->smpl_frm_shifts);
+ free(stats->nvaf);
+ free(stats->dvaf);
}
for (j=0; j<args->nusr; j++) free(args->usr[j].tag);
if ( args->af_bins ) bin_destroy(args->af_bins);
}
}
+static inline void update_dvaf(stats_t *stats, bcf1_t *line, bcf_fmt_t *fmt, int ismpl, int ial, int jal)
+{
+ if ( !fmt ) return;
+
+ float dvaf;
+ #define BRANCH_INT(type_t,missing,vector_end) { \
+ type_t *p = (type_t *) (fmt->p + fmt->size*ismpl); \
+ if ( p[ial]==vector_end || p[jal]==vector_end ) return; \
+ if ( p[ial]==missing || p[jal]==missing ) return; \
+ if ( !p[ial] && !p[jal] ) return; \
+ dvaf = (float)p[ial]/(p[ial]+p[jal]); \
+ }
+ switch (fmt->type) {
+ case BCF_BT_INT8: BRANCH_INT(int8_t, bcf_int8_missing, bcf_int8_vector_end); break;
+ case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_missing, bcf_int16_vector_end); break;
+ case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_missing, bcf_int32_vector_end); break;
+ default: fprintf(bcftools_stderr, "[E::%s] todo: %d\n", __func__, fmt->type); exit(1); break;
+ }
+ #undef BRANCH_INT
+
+ int len = line->d.var[ial].n;
+ if ( len < -stats->m_indel ) len = -stats->m_indel;
+ else if ( len > stats->m_indel ) len = stats->m_indel;
+ int bin = stats->m_indel + len;
+ stats->nvaf[bin]++;
+ stats->dvaf[bin] += dvaf;
+}
+
static void do_sample_stats(args_t *args, stats_t *stats, bcf_sr_t *reader, int matched)
{
bcf_srs_t *files = args->files;
if ( (fmt_ptr = bcf_get_fmt(reader->header,reader->buffer[0],"GT")) )
{
+ bcf_fmt_t *ad_fmt_ptr = bcf_get_variant_types(line)&VCF_INDEL ? bcf_get_fmt(reader->header,reader->buffer[0],"AD") : NULL;
+
int ref = bcf_acgt2int(*line->d.allele[0]);
int is, n_nref = 0, i_nref = 0;
for (is=0; is<args->files->n_smpl; is++)
if ( gt != GT_HOM_RR )
{
stats->smpl_indels[is]++;
- if ( gt==GT_HET_RA || gt==GT_HET_AA ) stats->smpl_indel_hets[is]++;
- else if ( gt==GT_HOM_AA ) stats->smpl_indel_homs[is]++;
+
+ if ( gt==GT_HET_RA || gt==GT_HET_AA )
+ {
+ int is_ins = 0, is_del = 0;
+ if ( bcf_get_variant_type(line,ial)&VCF_INDEL )
+ {
+ if ( line->d.var[ial].n < 0 ) is_del = 1;
+ else is_ins = 1;
+ update_dvaf(stats,line,ad_fmt_ptr,is,ial,jal);
+ }
+ if ( bcf_get_variant_type(line,jal)&VCF_INDEL )
+ {
+ if ( line->d.var[jal].n < 0 ) is_del = 1;
+ else is_ins = 1;
+ update_dvaf(stats,line,ad_fmt_ptr,is,jal,ial);
+ }
+ // Note that alt-het genotypes with both ins and del allele are counted twice!!
+ if ( is_del ) stats->smpl_del_hets[is]++;
+ if ( is_ins ) stats->smpl_ins_hets[is]++;
+ }
+ else if ( gt==GT_HOM_AA )
+ {
+ if ( line->d.var[ial].n < 0 ) stats->smpl_del_homs[is]++;
+ else stats->smpl_ins_homs[is]++;
+ }
}
if ( stats->smpl_frm_shifts )
{
}
#undef BRANCH_INT
}
+ else if ( (fmt_ptr = bcf_get_fmt(reader->header,reader->buffer[0],"AD")) )
+ {
+ #define BRANCH_INT(type_t,missing,vector_end) { \
+ int is,iv; \
+ for (is=0; is<args->files->n_smpl; is++) \
+ { \
+ type_t *p = (type_t *) (fmt_ptr->p + fmt_ptr->size*is); \
+ int dp = 0, has_value = 0; \
+ for (iv=0; iv<fmt_ptr->n; iv++) \
+ { \
+ if ( p[iv]==vector_end ) break; \
+ if ( p[iv]==missing ) continue; \
+ has_value = 1; \
+ dp += p[iv]; \
+ } \
+ if ( has_value ) \
+ { \
+ (*idist(&stats->dp, dp))++; \
+ stats->smpl_ndp[is]++; \
+ stats->smpl_dp[is] += dp; \
+ } \
+ } \
+ }
+ switch (fmt_ptr->type) {
+ case BCF_BT_INT8: BRANCH_INT(int8_t, bcf_int8_missing, bcf_int8_vector_end); break;
+ case BCF_BT_INT16: BRANCH_INT(int16_t, bcf_int16_missing, bcf_int16_vector_end); break;
+ case BCF_BT_INT32: BRANCH_INT(int32_t, bcf_int32_missing, bcf_int32_vector_end); break;
+ default: fprintf(bcftools_stderr, "[E::%s] todo: %d\n", __func__, fmt_ptr->type); exit(1); break;
+ }
+ #undef BRANCH_INT
+ }
if ( matched==3 )
{
fmt1 = bcf_get_fmt(files->readers[1].header,files->readers[1].buffer[0],"GT"); if ( !fmt1 ) return;
// only the first ALT allele is considered
+ if (args->ntmp_iaf <= 1) return; // Do not consider invariate sites
int iaf = args->tmp_iaf[1];
int line_type = bcf_get_variant_types(files->readers[0].buffer[0]);
gtcmp_t *af_stats = line_type&VCF_SNP ? args->af_gts_snps : args->af_gts_indels;
{
nmm++;
bcf_sr_t *reader = &files->readers[0];
- fprintf(bcftools_stdout, "DBG\t%s\t%d\t%s\t%d\t%d\n",reader->header->id[BCF_DT_CTG][reader->buffer[0]->rid].key,reader->buffer[0]->pos+1,files->samples[is],gt,gt2);
+ fprintf(bcftools_stdout, "DBG\t%s\t%"PRId64"\t%s\t%d\t%d\n",reader->header->id[BCF_DT_CTG][reader->buffer[0]->rid].key,(int64_t) reader->buffer[0]->pos+1,files->samples[is],gt,gt2);
}
else
{
}
}
float nrd = nrefm+nmm ? 100.*nmm/(nrefm+nmm) : 0;
- fprintf(bcftools_stdout, "PSD\t%s\t%d\t%d\t%d\t%f\n", reader->header->id[BCF_DT_CTG][reader->buffer[0]->rid].key,reader->buffer[0]->pos+1,nm,nmm,nrd);
+ fprintf(bcftools_stdout, "PSD\t%s\t%"PRId64"\t%d\t%d\t%f\n", reader->header->id[BCF_DT_CTG][reader->buffer[0]->rid].key,(int64_t) reader->buffer[0]->pos+1,nm,nmm,nrd);
}
}
}
for (id=0; id<args->nstats; id++)
{
stats_t *stats = &args->stats[id];
- fprintf(bcftools_stdout, "SN\t%d\tnumber of records:\t%d\n", id, stats->n_records);
- fprintf(bcftools_stdout, "SN\t%d\tnumber of no-ALTs:\t%d\n", id, stats->n_noalts);
- fprintf(bcftools_stdout, "SN\t%d\tnumber of SNPs:\t%d\n", id, stats->n_snps);
- fprintf(bcftools_stdout, "SN\t%d\tnumber of MNPs:\t%d\n", id, stats->n_mnps);
- fprintf(bcftools_stdout, "SN\t%d\tnumber of indels:\t%d\n", id, stats->n_indels);
- fprintf(bcftools_stdout, "SN\t%d\tnumber of others:\t%d\n", id, stats->n_others);
- fprintf(bcftools_stdout, "SN\t%d\tnumber of multiallelic sites:\t%d\n", id, stats->n_mals);
- fprintf(bcftools_stdout, "SN\t%d\tnumber of multiallelic SNP sites:\t%d\n", id, stats->n_snp_mals);
+ fprintf(bcftools_stdout, "SN\t%d\tnumber of records:\t%u\n", id, stats->n_records);
+ fprintf(bcftools_stdout, "SN\t%d\tnumber of no-ALTs:\t%u\n", id, stats->n_noalts);
+ fprintf(bcftools_stdout, "SN\t%d\tnumber of SNPs:\t%u\n", id, stats->n_snps);
+ fprintf(bcftools_stdout, "SN\t%d\tnumber of MNPs:\t%u\n", id, stats->n_mnps);
+ fprintf(bcftools_stdout, "SN\t%d\tnumber of indels:\t%u\n", id, stats->n_indels);
+ fprintf(bcftools_stdout, "SN\t%d\tnumber of others:\t%u\n", id, stats->n_others);
+ fprintf(bcftools_stdout, "SN\t%d\tnumber of multiallelic sites:\t%u\n", id, stats->n_mals);
+ fprintf(bcftools_stdout, "SN\t%d\tnumber of multiallelic SNP sites:\t%u\n", id, stats->n_snp_mals);
}
fprintf(bcftools_stdout, "# TSTV, transitions/transversions:\n# TSTV\t[2]id\t[3]ts\t[4]tv\t[5]ts/tv\t[6]ts (1st ALT)\t[7]tv (1st ALT)\t[8]ts/tv (1st ALT)\n");
for (id=0; id<args->nstats; id++)
}
}
}
- fprintf(bcftools_stdout, "# IDD, InDel distribution:\n# IDD\t[2]id\t[3]length (deletions negative)\t[4]count\n");
+ fprintf(bcftools_stdout, "# IDD, InDel distribution:\n# IDD\t[2]id\t[3]length (deletions negative)\t[4]number of sites\t[5]number of genotypes\t[6]mean VAF\n");
for (id=0; id<args->nstats; id++)
{
stats_t *stats = &args->stats[id];
for (i=stats->m_indel-1; i>=0; i--)
- if ( stats->deletions[i] ) fprintf(bcftools_stdout, "IDD\t%d\t%d\t%d\n", id,-i-1,stats->deletions[i]);
+ {
+ if ( !stats->deletions[i] ) continue;
+ // whops, differently organized arrow, dels are together with ins
+ int bin = stats->m_indel - i - 1;
+ fprintf(bcftools_stdout, "IDD\t%d\t%d\t%d\t", id,-i-1,stats->deletions[i]);
+ if ( stats->nvaf && stats->nvaf[bin] )
+ fprintf(bcftools_stdout, "%u\t%.2f",stats->nvaf[bin],stats->dvaf[bin]/stats->nvaf[bin]);
+ else
+ fprintf(bcftools_stdout, "0\t.");
+ fprintf(bcftools_stdout, "\n");
+ }
for (i=0; i<stats->m_indel; i++)
- if ( stats->insertions[i] ) fprintf(bcftools_stdout, "IDD\t%d\t%d\t%d\n", id,i+1,stats->insertions[i]);
+ {
+ if ( !stats->insertions[i] ) continue;
+ int bin = stats->m_indel + i + 1;
+ fprintf(bcftools_stdout, "IDD\t%d\t%d\t%d\t", id,i+1,stats->insertions[i]);
+ if ( stats->nvaf && stats->nvaf[bin] )
+ fprintf(bcftools_stdout, "%u\t%.2f",stats->nvaf[bin],stats->dvaf[bin]/stats->nvaf[bin]);
+ else
+ fprintf(bcftools_stdout, "0\t.");
+ fprintf(bcftools_stdout, "\n");
+ }
}
fprintf(bcftools_stdout, "# ST, Substitution types:\n# ST\t[2]id\t[3]type\t[4]count\n");
for (id=0; id<args->nstats; id++)
}
}
-
- fprintf(bcftools_stdout, "# PSI, Per-Sample Indels\n# PSI\t[2]id\t[3]sample\t[4]in-frame\t[5]out-frame\t[6]not applicable\t[7]out/(in+out) ratio\t[8]nHets\t[9]nAA\n");
+ fprintf(bcftools_stdout, "# PSI, Per-Sample Indels. Note that alt-het genotypes with both ins and del allele are counted twice, in both nInsHets and nDelHets.\n");
+ fprintf(bcftools_stdout, "# PSI\t[2]id\t[3]sample\t[4]in-frame\t[5]out-frame\t[6]not applicable\t[7]out/(in+out) ratio\t[8]nInsHets\t[9]nDelHets\t[10]nInsAltHoms\t[11]nDelAltHoms\n");
for (id=0; id<args->nstats; id++)
{
stats_t *stats = &args->stats[id];
in = stats->smpl_frm_shifts[i*3 + 1];
out = stats->smpl_frm_shifts[i*3 + 2];
}
- int nhom = stats->smpl_indel_homs[i];
- int nhet = stats->smpl_indel_hets[i];
- fprintf(bcftools_stdout, "PSI\t%d\t%s\t%d\t%d\t%d\t%.2f\t%d\t%d\n", id,args->files->samples[i], in,out,na,in+out?1.0*out/(in+out):0,nhet,nhom);
+ fprintf(bcftools_stdout, "PSI\t%d\t%s\t%d\t%d\t%d\t%.2f\t%d\t%d\t%d\t%d\n", id,args->files->samples[i], in,out,na,in+out?1.0*out/(in+out):0,
+ stats->smpl_ins_hets[i],stats->smpl_del_hets[i],stats->smpl_ins_homs[i],stats->smpl_del_homs[i]);
}
}
fprintf(bcftools_stderr, " -t, --targets <region> similar to -r but streams rather than index-jumps\n");
fprintf(bcftools_stderr, " -T, --targets-file <file> similar to -R but streams rather than index-jumps\n");
fprintf(bcftools_stderr, " -u, --user-tstv <TAG[:min:max:n]> collect Ts/Tv stats for any tag using the given binning [0:1:100]\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra decompression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(bcftools_stderr, " -v, --verbose produce verbose per-site and per-sample output\n");
fprintf(bcftools_stderr, "\n");
exit(1);
case 'i': args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 'h':
- case '?': usage();
+ case '?': usage(); break;
default: error("Unknown argument: %s\n", optarg);
}
}
while (fname)
{
if ( !bcf_sr_add_reader(args->files, fname) )
- error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
fname = ++optind < argc ? argv[optind] : NULL;
}
#include <sys/stat.h>
#include <sys/types.h>
#include <math.h>
+#include <inttypes.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/vcfutils.h>
if (args->calc_ac && args->update_info)
{
- bcf_hdr_append(args->hdr,"##INFO=<ID=AC,Number=A,Type=Integer,Description=\"Allele count in genotypes\">");
- bcf_hdr_append(args->hdr,"##INFO=<ID=AN,Number=1,Type=Integer,Description=\"Total number of alleles in called genotypes\">");
+ if (bcf_hdr_append(args->hdr,"##INFO=<ID=AC,Number=A,Type=Integer,Description=\"Allele count in genotypes\">") < 0)
+ error_errno("[%s] Failed to add \"AC\" INFO header", __func__);
+ if (bcf_hdr_append(args->hdr,"##INFO=<ID=AN,Number=1,Type=Integer,Description=\"Total number of alleles in called genotypes\">") < 0)
+ error_errno("[%s] Failed to add \"AN\" INFO header", __func__);
}
if (args->record_cmd_line) bcf_hdr_append_version(args->hdr, args->argc, args->argv, "bcftools_view");
- else bcf_hdr_sync(args->hdr);
+ else if (bcf_hdr_sync(args->hdr) < 0)
+ error_errno("[%s] Failed to update header", __func__);
// setup sample data
if (args->sample_names)
if (args->trim_alts)
{
int ret = bcf_trim_alleles(args->hsub ? args->hsub : args->hdr, line);
- if ( ret<0 ) error("Error: Could not trim alleles at %s:%d\n", bcf_seqname(args->hsub ? args->hsub : args->hdr, line), line->pos+1);
+ if ( ret<0 ) error("Error: Could not trim alleles at %s:%"PRId64"\n", bcf_seqname(args->hsub ? args->hsub : args->hdr, line), (int64_t) line->pos+1);
}
if (args->phased) {
int phased = bcf_all_phased(args->hdr, line);
fprintf(stderr, " -R, --regions-file <file> restrict to regions listed in a file\n");
fprintf(stderr, " -t, --targets [^]<region> similar to -r but streams rather than index-jumps. Exclude regions with \"^\" prefix\n");
fprintf(stderr, " -T, --targets-file [^]<file> similar to -R but streams rather than index-jumps. Exclude regions with \"^\" prefix\n");
- fprintf(stderr, " --threads <int> number of extra (de)compression threads [0]\n");
+ fprintf(stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(stderr, "\n");
fprintf(stderr, "Subset options:\n");
- fprintf(stderr, " -a, --trim-alt-alleles trim alternate alleles not seen in the subset\n");
+ fprintf(stderr, " -a, --trim-alt-alleles trim ALT alleles not seen in the genotype fields (or their subset with -s/-S)\n");
fprintf(stderr, " -I, --no-update do not (re)calculate INFO fields for the subset (currently INFO/AC and INFO/AN)\n");
fprintf(stderr, " -s, --samples [^]<list> comma separated list of samples to include (or exclude with \"^\" prefix)\n");
fprintf(stderr, " -S, --samples-file [^]<file> file of samples to include (or exclude with \"^\" prefix)\n");
}
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
- case '?': usage(args);
+ case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
}
}
if ( bcf_sr_set_threads(args->files, args->n_threads)<0 ) error("Failed to create threads\n");
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
init_data(args);
bcf_hdr_t *out_hdr = args->hnull ? args->hnull : (args->hsub ? args->hsub : args->hdr);
if (args->print_header)
- bcf_hdr_write(args->out, out_hdr);
+ {
+ if ( bcf_hdr_write(args->out, out_hdr)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out);
+ }
else if ( args->output_type & FT_BCF )
error("BCF output requires header, cannot proceed with -H\n");
{
bcf1_t *line = args->files->readers[0].buffer[0];
if ( line->errcode && out_hdr!=args->hdr ) error("Undefined tags in the header, cannot proceed in the sample subset mode.\n");
- if ( subset_vcf(args, line) )
- bcf_write1(args->out, out_hdr, line);
+ if ( subset_vcf(args, line) && bcf_write1(args->out, out_hdr, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out);
}
ret = args->files->errnum;
if ( ret ) fprintf(stderr,"Error: %s\n", bcf_sr_strerror(args->files->errnum));
#include <sys/stat.h>
#include <sys/types.h>
#include <math.h>
+#include <inttypes.h>
#include <htslib/vcf.h>
#include <htslib/synced_bcf_reader.h>
#include <htslib/vcfutils.h>
if (args->calc_ac && args->update_info)
{
- bcf_hdr_append(args->hdr,"##INFO=<ID=AC,Number=A,Type=Integer,Description=\"Allele count in genotypes\">");
- bcf_hdr_append(args->hdr,"##INFO=<ID=AN,Number=1,Type=Integer,Description=\"Total number of alleles in called genotypes\">");
+ if (bcf_hdr_append(args->hdr,"##INFO=<ID=AC,Number=A,Type=Integer,Description=\"Allele count in genotypes\">") < 0)
+ error_errno("[%s] Failed to add \"AC\" INFO header", __func__);
+ if (bcf_hdr_append(args->hdr,"##INFO=<ID=AN,Number=1,Type=Integer,Description=\"Total number of alleles in called genotypes\">") < 0)
+ error_errno("[%s] Failed to add \"AN\" INFO header", __func__);
}
if (args->record_cmd_line) bcf_hdr_append_version(args->hdr, args->argc, args->argv, "bcftools_view");
- else bcf_hdr_sync(args->hdr);
+ else if (bcf_hdr_sync(args->hdr) < 0)
+ error_errno("[%s] Failed to update header", __func__);
// setup sample data
if (args->sample_names)
if (args->trim_alts)
{
int ret = bcf_trim_alleles(args->hsub ? args->hsub : args->hdr, line);
- if ( ret<0 ) error("Error: Could not trim alleles at %s:%d\n", bcf_seqname(args->hsub ? args->hsub : args->hdr, line), line->pos+1);
+ if ( ret<0 ) error("Error: Could not trim alleles at %s:%"PRId64"\n", bcf_seqname(args->hsub ? args->hsub : args->hdr, line), (int64_t) line->pos+1);
}
if (args->phased) {
int phased = bcf_all_phased(args->hdr, line);
fprintf(bcftools_stderr, " -R, --regions-file <file> restrict to regions listed in a file\n");
fprintf(bcftools_stderr, " -t, --targets [^]<region> similar to -r but streams rather than index-jumps. Exclude regions with \"^\" prefix\n");
fprintf(bcftools_stderr, " -T, --targets-file [^]<file> similar to -R but streams rather than index-jumps. Exclude regions with \"^\" prefix\n");
- fprintf(bcftools_stderr, " --threads <int> number of extra (de)compression threads [0]\n");
+ fprintf(bcftools_stderr, " --threads <int> use multithreading with <int> worker threads [0]\n");
fprintf(bcftools_stderr, "\n");
fprintf(bcftools_stderr, "Subset options:\n");
- fprintf(bcftools_stderr, " -a, --trim-alt-alleles trim alternate alleles not seen in the subset\n");
+ fprintf(bcftools_stderr, " -a, --trim-alt-alleles trim ALT alleles not seen in the genotype fields (or their subset with -s/-S)\n");
fprintf(bcftools_stderr, " -I, --no-update do not (re)calculate INFO fields for the subset (currently INFO/AC and INFO/AN)\n");
fprintf(bcftools_stderr, " -s, --samples [^]<list> comma separated list of samples to include (or exclude with \"^\" prefix)\n");
fprintf(bcftools_stderr, " -S, --samples-file [^]<file> file of samples to include (or exclude with \"^\" prefix)\n");
}
case 9 : args->n_threads = strtol(optarg, 0, 0); break;
case 8 : args->record_cmd_line = 0; break;
- case '?': usage(args);
+ case '?': usage(args); break;
default: error("Unknown argument: %s\n", optarg);
}
}
}
if ( bcf_sr_set_threads(args->files, args->n_threads)<0 ) error("Failed to create threads\n");
- if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to open %s: %s\n", fname,bcf_sr_strerror(args->files->errnum));
+ if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
init_data(args);
bcf_hdr_t *out_hdr = args->hnull ? args->hnull : (args->hsub ? args->hsub : args->hdr);
if (args->print_header)
- bcf_hdr_write(args->out, out_hdr);
+ {
+ if ( bcf_hdr_write(args->out, out_hdr)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out);
+ }
else if ( args->output_type & FT_BCF )
error("BCF output requires header, cannot proceed with -H\n");
{
bcf1_t *line = args->files->readers[0].buffer[0];
if ( line->errcode && out_hdr!=args->hdr ) error("Undefined tags in the header, cannot proceed in the sample subset mode.\n");
- if ( subset_vcf(args, line) )
- bcf_write1(args->out, out_hdr, line);
+ if ( subset_vcf(args, line) && bcf_write1(args->out, out_hdr, line)!=0 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out);
}
ret = args->files->errnum;
if ( ret ) fprintf(bcftools_stderr,"Error: %s\n", bcf_sr_strerror(args->files->errnum));
#include <stdarg.h>
#include <stdlib.h>
#include <stdio.h>
+#include <errno.h>
#include <htslib/hts.h>
#include "bcftools.h"
#include "version.h"
exit(-1);
}
+void error_errno(const char *format, ...)
+{
+ va_list ap;
+ int e = errno;
+ va_start(ap, format);
+ vfprintf(stderr, format, ap);
+ va_end(ap);
+ if (e) {
+ fprintf(stderr, ": %s\n", strerror(e));
+ } else {
+ fprintf(stderr, "\n");
+ }
+ exit(-1);
+}
+
+
const char *hts_bcf_wmode(int file_type)
{
if ( file_type == FT_BCF ) return "wbu"; // uncompressed BCF
#include <stdarg.h>
#include <stdlib.h>
#include <stdio.h>
+#include <errno.h>
#include <htslib/hts.h>
#include "bcftools.h"
#include "version.h"
exit(-1);
}
+void error_errno(const char *format, ...)
+{
+ va_list ap;
+ int e = errno;
+ va_start(ap, format);
+ vfprintf(bcftools_stderr, format, ap);
+ va_end(ap);
+ if (e) {
+ fprintf(bcftools_stderr, ": %s\n", strerror(e));
+ } else {
+ fprintf(bcftools_stderr, "\n");
+ }
+ exit(-1);
+}
+
+
const char *hts_bcf_wmode(int file_type)
{
if ( file_type == FT_BCF ) return "wbu"; // uncompressed BCF
+++ /dev/null
-#define BCFTOOLS_VERSION "1.9"
--- /dev/null
+#!/bin/sh
+
+# Master version, for use in tarballs or non-git source copies
+VERSION=1.10.2
+
+# If we have a git clone, then check against the current tag
+if [ -e .git ]
+then
+ # If we ever get to 10.x this will need to be more liberal
+ VERSION=`git describe --match '[0-9].[0-9]*' --dirty --always`
+fi
+
+echo $VERSION
export HTSLIB_CONFIGURE_OPTIONS="--disable-libcurl"
PYBINS="/opt/python/*/bin"
+echo $PYBINS
+
for PYBIN in ${PYBINS}; do
${PYBIN}/pip install -r /io/requirements.txt
${PYBIN}/pip wheel /io/ -w wheelhouse/
# Bundle external shared libraries into the wheels
#
-# The '-L .' option is a workaround. By default, auditwheel puts all external
+# The '-L ""' option is a workaround. By default, auditwheel puts all external
# libraries (.so files) into a .libs directory and sets the RUNPATH to $ORIGIN/.libs.
# When HTSLIB_MODE is 'shared' (now the default), then all so libraries part of
# pysam require that RUNPATH is set to $ORIGIN (without the .libs). It seems
# auditwheel overwrites $ORIGIN with $ORIGIN/.libs. This workaround makes
-# auditwheel set the RUNPATH to "$ORIGIN/." and it will work as desired.
-#
+# auditwheel keeps the RUNPATH at "$ORIGIN" and put all the external libraries into
+# the pysam directory.
for whl in wheelhouse/*.whl; do
- auditwheel repair -L . $whl -w /io/wheelhouse/
+ auditwheel repair -L "" $whl -w /io/wheelhouse/
done
+# mkdir -p /io/wheelhouse
+# cp wheelhouse/*.whl /io/wheelhouse
+
# Created files are owned by root, so fix permissions.
chown -R --reference=/io/setup.py /io/wheelhouse/
# TODO Install packages and test them
-#for PYBIN in ${PYBINS}; do
-# ${PYBIN}/pip install pysam --no-index -f /io/wheelhouse
-# (cd $HOME; ${PYBIN}/nosetests ...)
-#done
+for PYBIN in ${PYBINS}; do
+ ${PYBIN}/pip install pysam --no-index -f /io/wheelhouse
+ # smoketest
+ (cd $HOME; ${PYBIN}/python -c 'import pysam')
+ # todo: add more tests
+done
#################################################################
-# Importing samtools and htslib
+# Importing samtools, bcftools, and htslib
#
-# For htslib, simply copy the whole release tar-ball
-# into the directory "htslib" and recreate the file version.h
+# For each package PKG:
#
-# rm -rf htslib
-# mv download/htslib htslib
-# git checkout -- htslib/version.h
-# Edit the file htslib/version.h to set the right version number.
-#
-# For samtools, type:
-# rm -rf samtools
-# python import.py samtools download/samtools
-# git checkout -- samtools/version.h
-#
-# Manually, then:
-# modify config.h to set compatibility flags
-#
-# For bcftools, type:
-# rm -rf bcftools
-# python import.py bcftools download/bedtools
-# git checkout -- bcftools/version.h
-# rm -rf bedtools/test bedtools/plugins
+# rm -rf PKG
+# python import.py PKG path/to/download/PKG-X.Y
import fnmatch
import os
EXCLUDE = {
"samtools": (
+ "test", "misc",
"razip.c",
"bgzip.c",
"main.c",
"calDepth.c",
"bam2bed.c",
- "wgsim.c",
"bam_tview.c",
"bam_tview.h",
"bam_tview_html.c",
"bam_tview_curses.c",
- "md5fa.c",
- "md5sum-lite.c",
- "maq2sam.c",
"bamcheck.c",
"chk_indel.c",
"vcf-miniview.c",
- "hfile_irods.c", # requires irods library
),
"bcftools": (
"test", "plugins", "peakfit.c",
"polysomy.c"),
"htslib": (
'htslib/tabix.c', 'htslib/bgzip.c',
- 'htslib/htsfile.c', 'htslib/hfile_irods.c'),
+ 'htslib/htsfile.c',
+ "test"),
}
}
+C_VERSION = {
+ "htslib": "HTS_VERSION_TEXT",
+ "samtools": "SAMTOOLS_VERSION",
+ "bcftools": "BCFTOOLS_VERSION"
+}
+
-def locate(pattern, root=os.curdir):
- '''Locate all files matching supplied filename pattern in and below
- supplied root directory.
+def locate(pattern, root=os.curdir, exclude=[], exclude_htslib=False):
+ '''Locate all files matching supplied filename pattern (but not listed
+ in exclude) in and below the supplied root directory. Omit any files under
+ directories listed in exclude or (if exclude_htslib=True) matching /htslib-*/.
'''
for path, dirs, files in os.walk(os.path.abspath(root)):
for filename in fnmatch.filter(files, pattern):
- yield os.path.join(path, filename)
+ if filename not in exclude:
+ yield os.path.join(path, filename)
+
+ for dirname in exclude:
+ if dirname in dirs: dirs.remove(dirname)
+ if exclude_htslib:
+ for dirname in [d for d in dirs if re.match(r"htslib-", d)]:
+ dirs.remove(dirname)
def _update_pysam_files(cf, destdir):
fn = os.path.basename(filename)
# some specific fixes:
SPECIFIC_SUBSTITUTIONS = {
+ "bamtk.c": (
+ 'else if (strcmp(argv[1], "tview") == 0)',
+ '//else if (strcmp(argv[1], "tview") == 0)'),
"bam_md.c": (
'sam_open_format("-", mode_w',
'sam_open_format({}_stdout_fn, mode_w'.format(basename)),
raise IOError(
"source directory `%s` does not exist." % srcdir)
- cfiles = locate("*.c", srcdir)
- hfiles = locate("*.h", srcdir)
- mfiles = itertools.chain(locate("README", srcdir), locate("LICENSE", srcdir))
-
- # remove unwanted files and htslib subdirectory.
- cfiles = [x for x in cfiles if os.path.basename(x) not in exclude
- and not re.search("htslib-", x)]
+ cfiles = locate("*.c", srcdir, exclude=exclude, exclude_htslib=True)
+ hfiles = locate("*.h", srcdir, exclude=exclude, exclude_htslib=True)
+ mfiles = itertools.chain(locate("README", srcdir), locate("LICENSE", srcdir),
+ locate("version.sh", srcdir, exclude_htslib=True))
- hfiles = [x for x in hfiles if os.path.basename(x) not in exclude
- and not re.search("htslib-", x)]
+ if dest == "htslib":
+ # Add build files, including *.ac *.in *.mk *.m4
+ mfiles = itertools.chain(mfiles, locate("Makefile", srcdir),
+ locate("configure", srcdir),
+ locate("*.[aim][cnk4]", srcdir))
ncopied = 0
sys.stdout.write(
"installed latest source code from %s: "
"%i files copied\n" % (srcdir, ncopied))
- # redirect stderr to pysamerr and replace bam.h with a stub.
- sys.stdout.write("applying stderr redirection\n")
- _update_pysam_files(cf, destdir)
+ if dest in MAIN:
+ # redirect stderr to pysamerr and replace bam.h with a stub.
+ sys.stdout.write("applying stderr redirection\n")
+
+ _update_pysam_files(cf, destdir)
+
+ def _getVersion(srcdir):
+ with open(os.path.join(srcdir, "version.sh"), encoding="utf-8") as inf:
+ for line in inf:
+ m = re.match(r"VERSION=(\S+)", line)
+ if m: return m.group(1)
+ raise ValueError("no VERSION line in version.sh")
+
+ def _update_version_file(key, value, filename):
+ tmpfilename = filename + ".tmp"
+ with open(filename, encoding="utf-8") as inf:
+ with open(tmpfilename, "w", encoding="utf-8") as outf:
+ for line in inf:
+ if key in line:
+ line = re.sub(r'"[^"]*"', '"{}"'.format(value), line)
+ outf.write(line)
+ os.rename(tmpfilename, filename)
+
+ version = _getVersion(srcdir)
+ _update_version_file("__{}_version__".format(dest), version, "pysam/version.py")
+ _update_version_file(C_VERSION[dest], version + " (pysam)", "pysam/version.h")
sys.exit(0)
#Install miniconda python
if [ $TRAVIS_OS_NAME == "osx" ]; then
- wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O Miniconda3.sh
+ wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O Miniconda3.sh
else
- wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O Miniconda3.sh --no-check-certificate # Default OS versions are old and have SSL / CERT issues
+ wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O Miniconda3.sh --no-check-certificate # Default OS versions are old and have SSL / CERT issues
fi
bash Miniconda3.sh -b
# NB: we force conda-forge:ncurses due to bioconda/bioconda-recipes#13488
conda install -y "samtools=1.9" "bcftools=1.9" "htslib=1.9" xz curl bzip2 conda-forge:ncurses
-# Need to make C compiler and linker use the anaconda includes and libraries:
-export PREFIX=~/miniconda3/
-export CFLAGS="-I${PREFIX}/include -L${PREFIX}/lib"
+# As HTSLIB_MODE is (defaulted to) 'shared', ensure we don't pick up
+# the external headers from the Conda-installed htslib package.
+mv $CONDA_PREFIX/include/htslib $CONDA_PREFIX/include/htslib-disable
+
export HTSLIB_CONFIGURE_OPTIONS="--disable-libcurl"
echo "show samtools, htslib, and bcftools versions"
A related issue is when different threads read from the same file
objec - or the same thread uses two iterators over a file. There is
only a single file-position for each opened file. To prevent this from
-hapeding, use the option ``mulitple_iterator=True`` when calling
+hapeding, use the option ``multiple_iterator=True`` when calling
a fetch() method. This will return an iterator on a newly opened
file.
The reason for this behaviour is that every iterator needs to keep
track of its current position in the file. Within pysam, each opened
file can only keep track of one file position and hence there can only
-be one iterator per file. Using the option ``mulitple_iterators=True``
+be one iterator per file. Using the option ``multiple_iterators=True``
will return an iterator within a newly opened file. This iterator will
not interfere with existing iterators as it has its own file handle
associated with it.
:meth:`~pysam.AlignmentFile.fetch` will only iterate over alignments
in the SAM/BAM file. The following thus always works::
- bf = pysam.AlignemFile(fname, "rb")
+ bf = pysam.AlignmentFile(fname, "rb")
for r in bf.fetch():
assert not r.is_unmapped
If the SAM/BAM file contains unaligned reads, they can be included
in the iteration by adding the ``until_eof=True`` flag::
- bf = pysam.AlignemFile(fname, "rb")
+ bf = pysam.AlignmentFile(fname, "rb")
for r in bf.fetch(until_eof=True):
if r.is_unmapped:
print ("read is unmapped")
iterating over a SAM/BAM file. To iterate over a file without
index, use the ``until_eof=True`::
- bf = pysam.AlignemFile(fname, "rb")
+ bf = pysam.AlignmentFile(fname, "rb")
for r in bf.fetch(until_eof=True):
print (r)
using cython and a high-level, pythonic API for convenient access to
the data within genomic file formats.
-The current version wraps *htslib-1.7*, *samtools-1.7* and
-*bcftools-1.6*.
+The current version wraps *htslib-1.10.2*, *samtools-1.10* and
+*bcftools-1.10.2*.
To install the latest release, type::
Release notes
=============
+Release 0.16.0
+==============
+
+This release wraps htslib/bcftools version 1.10.2 and samtools version
+1.10. The following bugs reported against pysam are fixed due to this:
+
+* [#447] Writing out QNAME longer than 251 characters corrupts BAM
+* [#640, #734, #843] Setting VariantRecord pos or stop raises error
+* [#738, #919] FastxFile truncates concatenated plain gzip compressed files
+
+Additional bugfixes:
+
+* [#840] Pileup doesn't work on python3 when `index_filename` is used
+* [#886] FastqProxy raises ValueError when instantiated from python
+* [#904] VariantFile.fetch() throws ValueError on files with no records
+* [#909] Fix incorrect quoting in VariantFile contig records
+* [#915, #916] Implement pileup() for unindexed files and/or SAM files
+
+Backwards incompatible changes:
+
+* The `samtools import` command was removed in samtools 1.10, so pysam
+ no longer exports a `samimport` function. Use `pysam.view()` instead.
+
+
Release 0.15.4
==============
"index",
"annotate",
"concat",
+ "convert",
"isec",
"merge",
"norm",
"plugin",
"query",
"reheader",
+ "sort",
"view",
"call",
"consensus",
"cnv",
+ "csq",
"filter",
"gtcheck",
+ "mpileup",
"roh",
"stats"]
#include "htslib/vcf.h"
#include "htslib/khash.h"
-int hts_useek(htsFile *fp, long uoffset, int where);
-long hts_utell(htsFile *fp);
-
int hts_set_verbosity(int verbosity);
int hts_get_verbosity(void);
add_indels : bool
- If True, add bases for bases inserted into the reference and
- 'N's for base skipped from the reference. If a reference sequence
- is given, add the actual bases.
+ If True, add bases for bases inserted into or skipped from the
+ reference. The latter requires a reference sequence file to have
+ been given, e.g. via `pileup(fastafile = ...)`. If no reference
+ sequence is available, skipped bases are represented as 'N's.
Returns
-------
cdef _setup_iterator(self,
int tid,
int start,
- int stop,
- int multiple_iterators=?)
+ int stop,
+ int multiple_iterators=?)
+ cdef _setup_raw_rest_iterator(self)
cdef reset(self, tid, start, stop)
cdef _free_pileup_iter(self)
# backwards compatibility
cdef char * getSequence(self)
-
+
cdef class IteratorColumnRegion(IteratorColumn):
cdef int start
pass
+cdef class IteratorColumnAll(IteratorColumn):
+ pass
+
+
cdef class IndexedReads:
cdef AlignmentFile samfile
cdef htsFile * htsfile
# class IteratorRowSelection
# class IteratorColumn
# class IteratorColumnRegion
+# class IteratorColumnAll
# class IteratorColumnAllRefs
#
########################################################
existing base qualities. The default is False (use existing
base qualities).
- adjust_capq_threshold: int
-
- adjust mapping quality. The default is 0 for no
- adjustment. The recommended value for adjustment is 50.
-
Returns
-------
has_coord, rtid, rstart, rstop = self.parse_region(
contig, start, stop, region, reference=reference, end=end)
- if self.is_bam or self.is_cram:
+ if has_coord:
if not self.has_index():
raise ValueError("no index available for pileup")
- if has_coord:
- return IteratorColumnRegion(self,
- tid=rtid,
- start=rstart,
- stop=rstop,
- **kwargs)
- else:
- return IteratorColumnAllRefs(self, **kwargs)
-
+ return IteratorColumnRegion(self,
+ tid=rtid,
+ start=rstart,
+ stop=rstop,
+ **kwargs)
else:
- raise NotImplementedError(
- "pileup of samfiles not implemented yet")
+ if self.has_index():
+ return IteratorColumnAllRefs(self, **kwargs)
+ else:
+ return IteratorColumnAll(self, **kwargs)
def count(self,
contig=None,
end of the genomic region (0-based exclusive). If not given,
count to the end of the chromosome.
- region : int
+ region : string
a region string.
quality_threshold : int
if samfile.has_index():
if samfile.index_filename:
- cindexname = samfile.index_filename
+ cindexname = bindex_filename = encode_filename(samfile.index_filename)
with nogil:
self.index = sam_index_load2(self.htsfile, cfilename, cindexname)
else:
return ret
+cdef int __advance_raw_nofilter(void *data, bam1_t *b):
+ '''advance (without iterator) without any read filtering.
+ '''
+ cdef __iterdata * d = <__iterdata*>data
+ cdef int ret
+ with nogil:
+ ret = sam_read1(d.htsfile, d.header, b)
+ return ret
+
+
cdef int __advance_all(void *data, bam1_t *b):
'''only use reads for pileup passing basic filters such as
return ret
+cdef int __advance_raw_all(void *data, bam1_t *b):
+ '''only use reads for pileup passing basic filters such as
+
+ BAM_FUNMAP, BAM_FSECONDARY, BAM_FQCFAIL, BAM_FDUP
+ '''
+
+ cdef __iterdata * d = <__iterdata*>data
+ cdef int ret
+ while 1:
+ with nogil:
+ ret = sam_read1(d.htsfile, d.header, b)
+ if ret < 0:
+ break
+ if b.core.flag & d.flag_filter:
+ continue
+ break
+ return ret
+
+
cdef int __advance_samtools(void * data, bam1_t * b):
'''advance using same filter and read processing as in
the samtools pileup.
while 1:
with nogil:
- ret = sam_itr_next(d.htsfile, d.iter, b)
+ ret = sam_itr_next(d.htsfile, d.iter, b) if d.iter else sam_read1(d.htsfile, d.header, b)
if ret < 0:
break
if b.core.flag & d.flag_filter:
with nogil:
bam_mplp_init_overlaps(self.pileup_iter)
+ cdef _setup_raw_rest_iterator(self):
+ '''set up an "iterator" that just uses sam_read1(), similar to HTS_IDX_REST'''
+
+ self.iter = None
+ self.iterdata.iter = NULL
+ self.iterdata.htsfile = self.samfile.htsfile
+ self.iterdata.seq = NULL
+ self.iterdata.tid = -1
+ self.iterdata.header = self.samfile.header.ptr
+
+ if self.fastafile is not None:
+ self.iterdata.fastafile = self.fastafile.fastafile
+ else:
+ self.iterdata.fastafile = NULL
+
+ # Free any previously allocated memory before reassigning
+ # pileup_iter
+ self._free_pileup_iter()
+
+ cdef void * data[1]
+ data[0] = <void*>&self.iterdata
+
+ if self.stepper is None or self.stepper == "all":
+ with nogil:
+ self.pileup_iter = bam_mplp_init(1,
+ <bam_plp_auto_f>&__advance_raw_all,
+ data)
+ elif self.stepper == "nofilter":
+ with nogil:
+ self.pileup_iter = bam_mplp_init(1,
+ <bam_plp_auto_f>&__advance_raw_nofilter,
+ data)
+ elif self.stepper == "samtools":
+ with nogil:
+ self.pileup_iter = bam_mplp_init(1,
+ <bam_plp_auto_f>&__advance_samtools,
+ data)
+ else:
+ raise ValueError(
+ "unknown stepper option `%s` in IteratorColumn" % self.stepper)
+
+ if self.max_depth:
+ with nogil:
+ bam_mplp_set_maxcnt(self.pileup_iter, self.max_depth)
+
+ if self.ignore_overlaps:
+ with nogil:
+ bam_mplp_init_overlaps(self.pileup_iter)
+
cdef reset(self, tid, start, stop):
'''reset iterator position.
This permits using the iterator multiple times without
having to incur the full set-up costs.
'''
+ if self.iter is None:
+ raise TypeError("Raw iterator set up without region cannot be reset")
+
self.iter = IteratorRowRegion(self.samfile, tid, start, stop, multiple_iterators=0)
self.iterdata.iter = self.iter.iter
self.samfile.header)
+cdef class IteratorColumnAll(IteratorColumn):
+ """iterates over all columns, without using an index
+ """
+
+ def __cinit__(self,
+ AlignmentFile samfile,
+ **kwargs):
+
+ self._setup_raw_rest_iterator()
+
+ def __next__(self):
+
+ cdef int n
+ n = self.cnext()
+ if n < 0:
+ raise ValueError("error during iteration")
+
+ if n == 0:
+ raise StopIteration
+
+ return makePileupColumn(&self.plp,
+ self.tid,
+ self.pos,
+ self.n_plp,
+ self.min_base_quality,
+ self.iterdata.seq,
+ self.samfile.header)
+
+
cdef class SNPCall:
'''the results of a SNP call.'''
cdef int _tid
from pysam.libchtslib cimport HTSFile, hisremote
+from pysam.utils import unquoted_str
+
__all__ = ['VariantFile',
'VariantHeader',
if type is not None:
raise ValueError('Type must be None when adding a filter')
- items = [('ID', id), ('Description', description)]
+ items = [('ID', unquoted_str(id)), ('Description', description)]
else:
if type not in VALUE_TYPES:
raise ValueError('unknown type specified: {}'.format(type))
if number is None:
number = '.'
- items = [('ID', id),
- ('Number', number),
- ('Type', type),
+ items = [('ID', unquoted_str(id)),
+ ('Number', unquoted_str(number)),
+ ('Type', unquoted_str(type)),
('Description', description)]
items += kwargs.items()
#TODO: implement __richcmp__
- def add(self, id, **kwargs):
+ def add(self, id, length=None, **kwargs):
"""Add a new contig record"""
if id in self:
raise ValueError('Header already exists for contig {}'.format(id))
- items = [('ID', id)]
+ items = [('ID', unquoted_str(id))]
+ if length is not None:
+ items.append(("length", unquoted_str(length)))
items += kwargs.items()
self.header.add_meta('contig', items=items)
hrec.value = strdup(force_bytes(value))
else:
for key, value in items:
-
- quoted = True
- if key in set(("ID", "Number", "Type")):
- quoted = False
+ quoted = not isinstance(value, unquoted_str) and key not in ("ID", "Number", "Type")
key = force_bytes(key)
bcf_hrec_add_key(hrec, key, <int>len(key))
value = force_bytes(str(value))
-
bcf_hrec_set_val(hrec, hrec.nkeys-1, value, <int>len(value), quoted)
except:
bcf_hrec_destroy(hrec)
bcf.seek(self.start_offset)
return iter(bcf)
- if not self.index:
+ if self.index is None:
raise ValueError('fetch requires an index')
_, tid, start, stop = self.parse_region(contig, start, stop, region,
cdef class FastqProxy:
"""A single entry in a fastq file."""
- def __init__(self): pass
+ def __init__(self):
+ raise ValueError("do not instantiate FastqProxy directly")
property name:
"""The name of each entry in the fastq file."""
int8_t HTS_FMT_CRAI
BGZF *hts_get_bgzfp(htsFile *fp)
- int hts_useek(htsFile *fp, long uoffset, int where)
- long hts_utell(htsFile *fp)
ctypedef struct hts_idx_t
with nogil:
ret = bgzf_seek(hts_get_bgzfp(self.htsfile), offset, SEEK_SET)
elif self.htsfile.format.compression == no_compression:
- with nogil:
- ret = hts_useek(self.htsfile, <int>offset, SEEK_SET)
+ ret = 0 if (hseek(self.htsfile.fp.hfile, offset, SEEK_SET) >= 0) else -1
else:
raise NotImplementedError("seek not implemented in files compressed by method {}".format(
self.htsfile.format.compression))
with nogil:
ret = bgzf_tell(hts_get_bgzfp(self.htsfile))
elif self.htsfile.format.compression == no_compression:
- with nogil:
- ret = hts_utell(self.htsfile)
+ ret = htell(self.htsfile.fp.hfile)
elif self.htsfile.format.format == cram:
with nogil:
ret = htell(cram_fd_get_fp(self.htsfile.fp.cram))
from pysam.utils import PysamDispatcher
# samtools command line options to export in python
-#
-# import is a python reserved word.
SAMTOOLS_DISPATCH = {
# samtools 'documented' commands
"view": ("view", None),
"mpileup": ("mpileup", None),
"depth": ("depth", None),
"faidx": ("faidx", None),
+ "fqidx": ("fqidx", None),
"tview": ("tview", None),
"index": ("index", None),
"idxstats": ("idxstats", None),
"flagstat": ("flagstat", None),
"calmd": ("calmd", None),
"merge": ("merge", None),
+ "markdup": ("markdup", None),
"rmdup": ("rmdup", None),
"reheader": ("reheader", None),
"cat": ("cat", None),
"targetcut": ("targetcut", None),
"phase": ("phase", None),
- "samimport": ("import", None),
"bam2fq": ("bam2fq", None),
"dict": ("dict", None),
"addreplacerg": ("addreplacerg", None),
"pad2unpad": ("pad2unpad", None),
"depad": ("pad2unpad", None),
"bedcov": ("bedcov", None),
+ "coverage": ("coverage", None),
"bamshuf": ("bamshuf", None),
"collate": ("collate", None),
"stats": ("stats", None),
"fastq": ("fastq", None),
"quickcheck": ("quickcheck", None),
"split": ("split", None),
+ "flags": ("flags", None),
}
# instantiate samtools commands as python functions
return stderr
else:
return stdout
+
+
+class unquoted_str(str):
+ '''Tag a value as an unquoted string. Meta-information in the VCF
+ header takes the form of key=value pairs. By default, pysam will
+ enclose the value in quotation marks. Tagging that value with
+ unquoted_str will prevent this quoting.'''
--- /dev/null
+// Version information used while compiling samtools, bcftools, and htslib
+
+#define SAMTOOLS_VERSION "1.10 (pysam)"
+#define BCFTOOLS_VERSION "1.10.2 (pysam)"
+#define HTS_VERSION_TEXT "1.10.2 (pysam)"
# pysam versioning information
-__version__ = "0.15.4"
+__version__ = "0.16.0.1"
-# TODO: upgrade number
-__samtools_version__ = "1.9"
-
-# TODO: upgrade code and number
-__bcftools_version__ = "1.9"
-
-__htslib_version__ = "1.9"
+__samtools_version__ = "1.10"
+__bcftools_version__ = "1.10.2"
+__htslib_version__ = "1.10.2"
The MIT/Expat License
-Copyright (C) 2008-2018 Genome Research Ltd.
+Copyright (C) 2008-2019 Genome Research Ltd.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
The typical simple case of building Samtools using the HTSlib bundled within
this Samtools release tarball is done as follows:
- cd .../samtools-1.9 # Within the unpacked release directory
+ cd .../samtools-1.10 # Within the unpacked release directory
./configure
make
installation using the HTSlib bundled within this Samtools release tarball,
and building the various HTSlib utilities such as bgzip is done as follows:
- cd .../samtools-1.9 # Within the unpacked release directory
+ cd .../samtools-1.10 # Within the unpacked release directory
./configure --prefix=/path/to/location
make all all-htslib
make install install-htslib
To build with plug-ins, you need to use the --enable-plugins configure option
as follows:
- cd .../samtools-1.9 # Within the unpacked release directory
+ cd .../samtools-1.10 # Within the unpacked release directory
./configure --enable-plugins --prefix=/path/to/location
make all all-htslib
make install install-htslib
the source distribution instead of installing the package. In that case
you can use:
- cd .../samtools-1.9 # Within the unpacked release directory
- ./configure --enable-plugins --with-plugin-path=$PWD/htslib-1.9
+ cd .../samtools-1.10 # Within the unpacked release directory
+ ./configure --enable-plugins --with-plugin-path=$PWD/htslib-1.10
make all all-htslib
It is possible to override the built-in search path using the HTS_PATH
/* bam.c -- BAM format.
- Copyright (C) 2008-2013, 2015 Genome Research Ltd.
+ Copyright (C) 2008-2013, 2015, 2019 Genome Research Ltd.
Portions copyright (C) 2009-2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include <errno.h>
#include "bam.h"
#include "htslib/kstring.h"
-#include "sam_header.h"
char *bam_format1(const bam_header_t *header, const bam1_t *b)
{
char *s;
if (b->core.tid < -1 || b->core.mtid < -1) return 0;
- if (header && (b->core.tid >= header->n_targets || b->core.mtid >= header->n_targets)) return 0;
+ if (header && (b->core.tid >= sam_hdr_nref(header) || b->core.mtid >= sam_hdr_nref(header))) return 0;
if (b->data_len < b->core.l_qname) return 0;
s = memchr(bam1_qname(b), '\0', b->core.l_qname);
// FIXME: we should also check the LB tag associated with each alignment
const char *bam_get_library(bam_header_t *h, const bam1_t *b)
{
- // Slow and inefficient. Rewrite once we get a proper header API.
const char *rg;
- char *cp = h->text;
+ kstring_t lib = { 0, 0, NULL };
rg = (char *)bam_aux_get(b, "RG");
if (!rg)
else
rg++;
- // Header is guaranteed to be nul terminated, so this is valid.
- while (*cp) {
- char *ID, *LB;
- char last = '\t';
-
- // Find a @RG line
- if (strncmp(cp, "@RG", 3) != 0) {
- while (*cp && *cp != '\n') cp++; // skip line
- if (*cp) cp++;
- continue;
- }
-
- // Find ID: and LB: keys
- cp += 4;
- ID = LB = NULL;
- while (*cp && *cp != '\n') {
- if (last == '\t') {
- if (strncmp(cp, "LB:", 3) == 0)
- LB = cp+3;
- else if (strncmp(cp, "ID:", 3) == 0)
- ID = cp+3;
- }
- last = *cp++;
- }
-
- if (!ID || !LB)
- continue;
+ if (sam_hdr_find_tag_id(h, "RG", "ID", rg, "LB", &lib) < 0)
+ return NULL;
- // Check it's the correct ID
- if (strncmp(rg, ID, strlen(rg)) != 0 || ID[strlen(rg)] != '\t')
- continue;
+ static char LB_text[1024];
+ int len = lib.l < sizeof(LB_text) - 1 ? lib.l : sizeof(LB_text) - 1;
- // Valid until next query
- static char LB_text[1024];
- for (cp = LB; *cp && *cp != '\t' && *cp != '\n'; cp++)
- ;
- strncpy(LB_text, LB, MIN(cp-LB, 1023));
- LB_text[MIN(cp-LB, 1023)] = 0;
+ memcpy(LB_text, lib.s, len);
+ LB_text[len] = 0;
- // Return it; valid until the next query.
- return LB_text;
- }
+ free(lib.s);
- return NULL;
+ return LB_text;
}
int bam_fetch(bamFile fp, const bam_index_t *idx, int tid, int beg, int end, void *data, bam_fetch_f func)
/* bam.c -- BAM format.
- Copyright (C) 2008-2013, 2015 Genome Research Ltd.
+ Copyright (C) 2008-2013, 2015, 2019 Genome Research Ltd.
Portions copyright (C) 2009-2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include <errno.h>
#include "bam.h"
#include "htslib/kstring.h"
-#include "sam_header.h"
char *bam_format1(const bam_header_t *header, const bam1_t *b)
{
char *s;
if (b->core.tid < -1 || b->core.mtid < -1) return 0;
- if (header && (b->core.tid >= header->n_targets || b->core.mtid >= header->n_targets)) return 0;
+ if (header && (b->core.tid >= sam_hdr_nref(header) || b->core.mtid >= sam_hdr_nref(header))) return 0;
if (b->data_len < b->core.l_qname) return 0;
s = memchr(bam1_qname(b), '\0', b->core.l_qname);
// FIXME: we should also check the LB tag associated with each alignment
const char *bam_get_library(bam_header_t *h, const bam1_t *b)
{
- // Slow and inefficient. Rewrite once we get a proper header API.
const char *rg;
- char *cp = h->text;
+ kstring_t lib = { 0, 0, NULL };
rg = (char *)bam_aux_get(b, "RG");
if (!rg)
else
rg++;
- // Header is guaranteed to be nul terminated, so this is valid.
- while (*cp) {
- char *ID, *LB;
- char last = '\t';
-
- // Find a @RG line
- if (strncmp(cp, "@RG", 3) != 0) {
- while (*cp && *cp != '\n') cp++; // skip line
- if (*cp) cp++;
- continue;
- }
-
- // Find ID: and LB: keys
- cp += 4;
- ID = LB = NULL;
- while (*cp && *cp != '\n') {
- if (last == '\t') {
- if (strncmp(cp, "LB:", 3) == 0)
- LB = cp+3;
- else if (strncmp(cp, "ID:", 3) == 0)
- ID = cp+3;
- }
- last = *cp++;
- }
-
- if (!ID || !LB)
- continue;
+ if (sam_hdr_find_tag_id(h, "RG", "ID", rg, "LB", &lib) < 0)
+ return NULL;
- // Check it's the correct ID
- if (strncmp(rg, ID, strlen(rg)) != 0 || ID[strlen(rg)] != '\t')
- continue;
+ static char LB_text[1024];
+ int len = lib.l < sizeof(LB_text) - 1 ? lib.l : sizeof(LB_text) - 1;
- // Valid until next query
- static char LB_text[1024];
- for (cp = LB; *cp && *cp != '\t' && *cp != '\n'; cp++)
- ;
- strncpy(LB_text, LB, MIN(cp-LB, 1023));
- LB_text[MIN(cp-LB, 1023)] = 0;
+ memcpy(LB_text, lib.s, len);
+ LB_text[len] = 0;
- // Return it; valid until the next query.
- return LB_text;
- }
+ free(lib.s);
- return NULL;
+ return LB_text;
}
int bam_fetch(bamFile fp, const bam_index_t *idx, int tid, int beg, int end, void *data, bam_fetch_f func)
/* bam.h -- BAM API.
- Copyright (C) 2008-2014 Genome Research Ltd.
+ Copyright (C) 2008-2014, 2019 Genome Research Ltd.
Portions copyright (C) 2010-2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
@copyright Genome Research Ltd.
*/
-#define BAM_VERSION "1.9"
+#define BAM_VERSION "1.10"
#include <stdint.h>
#include <stdlib.h>
*/
// int sam_read1(tamFile fp, bam_header_t *header, bam1_t *b);
- /*!
- @abstract Read header information from a TAB-delimited list file.
- @param fn_list file name for the list
- @return a pointer to the header structure
-
- @discussion Each line in this file consists of chromosome name and
- the length of chromosome.
- */
- bam_header_t *sam_header_read2(const char *fn_list);
-
/*!
@abstract Read header from a SAM file (if present)
@param fp SAM file handler
@abstract Initialize a header structure.
@return the pointer to the header structure
*/
- static inline bam_header_t *bam_header_init(void) { return bam_hdr_init(); }
+ static inline bam_header_t *bam_header_init(void) { return sam_hdr_init(); }
/*!
@abstract Destroy a header structure.
@param header pointer to the header
*/
- static inline void bam_header_destroy(bam_header_t *header) { bam_hdr_destroy(header); }
+ static inline void bam_header_destroy(bam_header_t *header) { sam_hdr_destroy(header); }
/*!
@abstract Read a header structure from BAM.
@param header pointer to the header structure
@return always 0 currently
*/
- static inline int bam_header_write(bamFile fp, const bam_header_t *header) { return bam_hdr_write(fp, header); }
+ static inline int bam_header_write(bamFile fp, bam_header_t *header) { return bam_hdr_write(fp, header); }
/*!
@abstract Read an alignment from BAM.
/* bam2bcf.c -- variant calling.
Copyright (C) 2010-2012 Broad Institute.
- Copyright (C) 2012-2014 Genome Research Ltd.
+ Copyright (C) 2012-2015 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
/* bam2bcf.c -- variant calling.
Copyright (C) 2010-2012 Broad Institute.
- Copyright (C) 2012-2014 Genome Research Ltd.
+ Copyright (C) 2012-2015 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
/* bam2bcf.h -- variant calling.
Copyright (C) 2010-2012 Broad Institute.
- Copyright (C) 2012-2014 Genome Research Ltd.
+ Copyright (C) 2012-2014, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
} bcf_callret1_t;
typedef struct {
- int tid, pos;
+ int tid;
+ hts_pos_t pos;
bcf_hdr_t *bcf_hdr;
int a[5]; // alleles: ref, alt, alt2, alt3
float qsum[5]; // for the QS tag
int bcf_call_combine(int n, const bcf_callret1_t *calls, bcf_callaux_t *bca, int ref_base /*4-bit*/, bcf_call_t *call);
int bcf_call2bcf(bcf_call_t *bc, bcf1_t *b, bcf_callret1_t *bcr, int fmt_flag,
const bcf_callaux_t *bca, const char *ref);
- int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_callaux_t *bca, const char *ref,
+ int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, hts_pos_t pos, bcf_callaux_t *bca, const char *ref,
const void *rghash);
void bcf_callaux_clean(bcf_callaux_t *bca, bcf_call_t *call);
/* bam2bcf_indel.c -- indel caller.
Copyright (C) 2010, 2011 Broad Institute.
- Copyright (C) 2012-2014 Genome Research Ltd.
+ Copyright (C) 2012-2014, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
kh_destroy(rg, hash);
}
-static int tpos2qpos(const bam1_core_t *c, const uint32_t *cigar, int32_t tpos, int is_left, int32_t *_tpos)
+static int tpos2qpos(const bam1_core_t *c, const uint32_t *cigar, hts_pos_t tpos, hts_pos_t is_left, hts_pos_t *_tpos)
{
- int k, x = c->pos, y = 0, last_y = 0;
+ int k, y = 0, last_y = 0;
+ hts_pos_t x = c->pos;
*_tpos = c->pos;
for (k = 0; k < c->n_cigar; ++k) {
int op = cigar[k] & BAM_CIGAR_MASK;
return q < qh? q : qh;
}
-static inline int est_indelreg(int pos, const char *ref, int l, char *ins4)
+static inline int est_indelreg(hts_pos_t pos, const char *ref, int l, char *ins4)
{
- int i, j, max = 0, max_i = pos, score = 0;
+ int j, max = 0, score = 0;
+ hts_pos_t i, max_i = pos;
l = abs(l);
for (i = pos + 1, j = 0; ref[i]; ++i, ++j) {
if (ins4) score += (toupper(ref[i]) != "ACGTN"[(int)ins4[j%l]])? -10 : 1;
- 8: estimated sequence quality .. (aux>>8)&0xff
- 8: indel quality .. aux&0xff
*/
-int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_callaux_t *bca, const char *ref,
+int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, hts_pos_t pos, bcf_callaux_t *bca, const char *ref,
const void *rghash)
{
- int i, s, j, k, t, n_types, *types, max_rd_len, left, right, max_ins, *score1, *score2, max_ref2;
+ int s, k, t, n_types, *types, max_rd_len, max_ins, *score1, *score2, max_ref2;
int N, K, l_run, ref_type, n_alt;
+ hts_pos_t i, j, left, right;
char *inscns = 0, *ref2, *query, **ref_sample;
khash_t(rg) *hash = (khash_t(rg)*)rghash;
if (ref == 0 || bca == 0) return -1;
free(aux);
// TODO revisit how/whether to control printing this warning
if (hts_verbose >= 2)
- fprintf(stderr, "[%s] excessive INDEL alleles at position %d. Skip the position.\n", __func__, pos + 1);
+ fprintf(stderr, "[%s] excessive INDEL alleles at position %"PRIhts_pos". Skip the position.\n", __func__, pos + 1);
return -1;
}
types = (int*)calloc(n_types, sizeof(int));
bam1_t *b = p->b;
uint32_t *cigar = bam_get_cigar(b);
uint8_t *seq = bam_get_seq(b);
- int x = b->core.pos, y = 0;
+ hts_pos_t x = b->core.pos, y = 0;
for (k = 0; k < b->core.n_cigar; ++k) {
int op = cigar[k]&0xf;
int j, l = cigar[k]>>4;
// align each read to ref2
for (i = 0; i < n_plp[s]; ++i, ++K) {
bam_pileup1_t *p = plp[s] + i;
- int qbeg, qend, tbeg, tend, sc, kk;
+ int qbeg, qend, sc, kk;
+ hts_pos_t tbeg, tend;
uint8_t *seq = bam_get_seq(p->b);
uint32_t *cigar = bam_get_cigar(p->b);
if (p->b->core.flag&4) continue; // unmapped reads
/* bam2bcf_indel.c -- indel caller.
Copyright (C) 2010, 2011 Broad Institute.
- Copyright (C) 2012-2014 Genome Research Ltd.
+ Copyright (C) 2012-2014, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
kh_destroy(rg, hash);
}
-static int tpos2qpos(const bam1_core_t *c, const uint32_t *cigar, int32_t tpos, int is_left, int32_t *_tpos)
+static int tpos2qpos(const bam1_core_t *c, const uint32_t *cigar, hts_pos_t tpos, hts_pos_t is_left, hts_pos_t *_tpos)
{
- int k, x = c->pos, y = 0, last_y = 0;
+ int k, y = 0, last_y = 0;
+ hts_pos_t x = c->pos;
*_tpos = c->pos;
for (k = 0; k < c->n_cigar; ++k) {
int op = cigar[k] & BAM_CIGAR_MASK;
return q < qh? q : qh;
}
-static inline int est_indelreg(int pos, const char *ref, int l, char *ins4)
+static inline int est_indelreg(hts_pos_t pos, const char *ref, int l, char *ins4)
{
- int i, j, max = 0, max_i = pos, score = 0;
+ int j, max = 0, score = 0;
+ hts_pos_t i, max_i = pos;
l = abs(l);
for (i = pos + 1, j = 0; ref[i]; ++i, ++j) {
if (ins4) score += (toupper(ref[i]) != "ACGTN"[(int)ins4[j%l]])? -10 : 1;
- 8: estimated sequence quality .. (aux>>8)&0xff
- 8: indel quality .. aux&0xff
*/
-int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, int pos, bcf_callaux_t *bca, const char *ref,
+int bcf_call_gap_prep(int n, int *n_plp, bam_pileup1_t **plp, hts_pos_t pos, bcf_callaux_t *bca, const char *ref,
const void *rghash)
{
- int i, s, j, k, t, n_types, *types, max_rd_len, left, right, max_ins, *score1, *score2, max_ref2;
+ int s, k, t, n_types, *types, max_rd_len, max_ins, *score1, *score2, max_ref2;
int N, K, l_run, ref_type, n_alt;
+ hts_pos_t i, j, left, right;
char *inscns = 0, *ref2, *query, **ref_sample;
khash_t(rg) *hash = (khash_t(rg)*)rghash;
if (ref == 0 || bca == 0) return -1;
free(aux);
// TODO revisit how/whether to control printing this warning
if (hts_verbose >= 2)
- fprintf(samtools_stderr, "[%s] excessive INDEL alleles at position %d. Skip the position.\n", __func__, pos + 1);
+ fprintf(samtools_stderr, "[%s] excessive INDEL alleles at position %"PRIhts_pos". Skip the position.\n", __func__, pos + 1);
return -1;
}
types = (int*)calloc(n_types, sizeof(int));
bam1_t *b = p->b;
uint32_t *cigar = bam_get_cigar(b);
uint8_t *seq = bam_get_seq(b);
- int x = b->core.pos, y = 0;
+ hts_pos_t x = b->core.pos, y = 0;
for (k = 0; k < b->core.n_cigar; ++k) {
int op = cigar[k]&0xf;
int j, l = cigar[k]>>4;
// align each read to ref2
for (i = 0; i < n_plp[s]; ++i, ++K) {
bam_pileup1_t *p = plp[s] + i;
- int qbeg, qend, tbeg, tend, sc, kk;
+ int qbeg, qend, sc, kk;
+ hts_pos_t tbeg, tend;
uint8_t *seq = bam_get_seq(p->b);
uint32_t *cigar = bam_get_cigar(p->b);
if (p->b->core.flag&4) continue; // unmapped reads
/* bam2depth.c -- depth subcommand.
Copyright (C) 2011, 2012 Broad Institute.
- Copyright (C) 2012-2014 Genome Research Ltd.
+ Copyright (C) 2012-2016, 2018, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
#include <unistd.h>
#include "htslib/sam.h"
#include "samtools.h"
+#include "bedidx.h"
#include "sam_opts.h"
+#define BAM_FMAX ((BAM_FSUPPLEMENTARY << 1) - 1)
+
typedef struct { // auxiliary data structure
samFile *fp; // the file handle
- bam_hdr_t *hdr; // the file header
+ sam_hdr_t *hdr; // the file header
hts_itr_t *iter; // NULL if a region not specified
int min_mapQ, min_len; // mapQ filter; length filter
+ uint32_t flags; // read filtering flags
} aux_t;
-void *bed_read(const char *fn); // read a BED or position list file
-void bed_destroy(void *_h); // destroy the BED data structure
-int bed_overlap(const void *_h, const char *chr, int beg, int end); // test if chr:beg-end overlaps
-int bed_query(const void *_h, const char *chr, int pos, int *beg, int *end);
-
// This function reads a BAM alignment from one BAM file.
static int read_bam(void *data, bam1_t *b) // read level filters better go here to avoid pileup
{
{
ret = aux->iter? sam_itr_next(aux->fp, aux->iter, b) : sam_read1(aux->fp, aux->hdr, b);
if ( ret<0 ) break;
- if ( b->core.flag & (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP) ) continue;
+ if ( b->core.flag & aux->flags) continue;
if ( (int)b->core.qual < aux->min_mapQ ) continue;
if ( aux->min_len && bam_cigar2qlen(b->core.n_cigar, bam_get_cigar(b)) < aux->min_len ) continue;
break;
fprintf(stderr, " -a output all positions (including zero depth)\n");
fprintf(stderr, " -a -a (or -aa) output absolutely all positions, including unused ref. sequences\n");
fprintf(stderr, " -b <bed> list of positions or regions\n");
+ fprintf(stderr, " -X use customized index files\n");
fprintf(stderr, " -f <list> list of input BAM filenames, one per line [null]\n");
+ fprintf(stderr, " -H print a file header\n");
fprintf(stderr, " -l <int> read length threshold (ignore reads shorter than <int>) [0]\n");
fprintf(stderr, " -d/-m <int> maximum coverage depth [8000]. If 0, depth is set to the maximum\n"
" integer value, effectively removing any depth limit.\n"); // the htslib's default
+ fprintf(stderr, " -o FILE where to write output to [stdout]\n");
fprintf(stderr, " -q <int> base quality threshold [0]\n");
fprintf(stderr, " -Q <int> mapping quality threshold [0]\n");
fprintf(stderr, " -r <chr:from-to> region\n");
+ fprintf(stderr, " -g <flags> include reads that have any of the specified flags set [0]\n");
+ fprintf(stderr, " -G <flags> filter out reads that have any of the specified flags set"
+ " [UNMAP,SECONDARY,QCFAIL,DUP]\n");
- sam_global_opt_help(stderr, "-.--.-");
+ sam_global_opt_help(stderr, "-.--.--.");
fprintf(stderr, "\n");
fprintf(stderr, "The output is a simple tab-separated table with three columns: reference name,\n");
fprintf(stderr, "omitted by default; see the -a option.\n");
fprintf(stderr, "\n");
- return 1;
+ return EXIT_FAILURE;
}
int main_depth(int argc, char *argv[])
{
- int i, n, tid, reg_tid, beg, end, pos, *n_plp, baseQ = 0, mapQ = 0, min_len = 0;
+ int i, n, tid, reg_tid, *n_plp, baseQ = 0, mapQ = 0, min_len = 0, has_index_file = 0;
+ hts_pos_t beg, end, pos, last_pos = -1;
int all = 0, status = EXIT_SUCCESS, nfiles, max_depth = -1;
const bam_pileup1_t **plp;
char *reg = 0; // specified region
void *bed = 0; // BED data structure
char *file_list = NULL, **fn = NULL;
- bam_hdr_t *h = NULL; // BAM header of the 1st input
+ sam_hdr_t *h = NULL; // BAM header of the 1st input
aux_t **data;
bam_mplp_t mplp;
- int last_pos = -1, last_tid = -1, ret;
+ int last_tid = -1, ret;
+ int print_header = 0;
+ char *output_file = NULL;
+ FILE *file_out = stdout;
+ uint32_t flags = (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP);
+ int tflags = 0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
};
// parse the command line
- while ((n = getopt_long(argc, argv, "r:b:q:Q:l:f:am:d:", lopts, NULL)) >= 0) {
+ while ((n = getopt_long(argc, argv, "r:b:Xq:Q:l:f:am:d:Ho:g:G:", lopts, NULL)) >= 0) {
switch (n) {
case 'l': min_len = atoi(optarg); break; // minimum query length
case 'r': reg = strdup(optarg); break; // parsing a region requires a BAM header
case 'b':
bed = bed_read(optarg); // BED or position list file can be parsed now
- if (!bed) { print_error_errno("depth", "Could not read file \"%s\"", optarg); return 1; }
+ if (!bed) {
+ print_error_errno("depth", "Could not read file \"%s\"", optarg);
+ return EXIT_FAILURE;
+ }
break;
+ case 'X': has_index_file = 1; break;
case 'q': baseQ = atoi(optarg); break; // base quality threshold
case 'Q': mapQ = atoi(optarg); break; // mapping quality threshold
case 'f': file_list = optarg; break;
case 'a': all++; break;
case 'd': case 'm': max_depth = atoi(optarg); break; // maximum coverage depth
+ case 'H': print_header = 1; break;
+ case 'o': output_file = optarg; break;
+ case 'g':
+ tflags = bam_str2flag(optarg);
+ if (tflags < 0 || tflags > BAM_FMAX) {
+ print_error_errno("depth", "Flag value \"%s\" is not supported", optarg);
+ return 1;
+ }
+ flags &= ~tflags;
+ break;
+ case 'G':
+ tflags = bam_str2flag(optarg);
+ if (tflags < 0 || tflags > BAM_FMAX) {
+ print_error_errno("depth", "Flag value \"%s\" is not supported", optarg);
+ return 1;
+ }
+ flags |= tflags;
+ break;
default: if (parse_sam_global_opt(n, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': return usage();
if (optind == argc && !file_list)
return usage();
+ /* output file provided by user */
+ if (output_file != NULL && strcmp(output_file,"-")!=0) {
+ file_out = fopen( output_file, "w" );
+ if (file_out == NULL) {
+ print_error_errno("depth", "Cannot open \"%s\" for writing.", output_file);
+ return EXIT_FAILURE;
+ }
+ }
+
+
// initialize the auxiliary data structures
if (file_list)
{
- if ( read_file_list(file_list,&nfiles,&fn) ) return 1;
+ if (has_index_file) {
+ print_error("depth", "The -f option cannot be combined with -X");
+ return 1;
+ }
+ if ( read_file_list(file_list,&nfiles,&fn) ) return EXIT_FAILURE;
n = nfiles;
argv = fn;
optind = 0;
}
- else
- n = argc - optind; // the number of BAMs on the command line
+ else if (has_index_file) { // Calculate # of input BAM files
+ if ((argc - optind) % 2 != 0) {
+ fprintf(stderr, "Error: Odd number of filenames detected! Each BAM file should have an index file\n");
+ return 1;
+ }
+ n = (argc - optind) / 2;
+ } else {
+ n = argc - optind;
+ }
data = calloc(n, sizeof(aux_t*)); // data[i] for the i-th input
- reg_tid = 0; beg = 0; end = INT_MAX; // set the default region
+ reg_tid = 0; beg = 0; end = HTS_POS_MAX; // set the default region
+
for (i = 0; i < n; ++i) {
int rf;
data[i] = calloc(1, sizeof(aux_t));
rf = SAM_FLAG | SAM_RNAME | SAM_POS | SAM_MAPQ | SAM_CIGAR | SAM_SEQ;
if (baseQ) rf |= SAM_QUAL;
if (hts_set_opt(data[i]->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
- fprintf(stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
- return 1;
+ print_error_errno("depth", "Failed to set CRAM_OPT_REQUIRED_FIELDS value");
+ status = EXIT_FAILURE;
+ goto depth_end;
}
if (hts_set_opt(data[i]->fp, CRAM_OPT_DECODE_MD, 0)) {
- fprintf(stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
- return 1;
+ print_error_errno("depth", "Failed to set CRAM_OPT_DECODE_MD value");
+ status = EXIT_FAILURE;
+ goto depth_end;
}
data[i]->min_mapQ = mapQ; // set the mapQ filter
data[i]->min_len = min_len; // set the qlen filter
data[i]->hdr = sam_hdr_read(data[i]->fp); // read the BAM header
if (data[i]->hdr == NULL) {
- fprintf(stderr, "Couldn't read header for \"%s\"\n",
- argv[optind+i]);
+ print_error_errno("depth", "Couldn't read header for \"%s\"",
+ argv[optind+i]);
status = EXIT_FAILURE;
goto depth_end;
}
if (reg) { // if a region is specified
- hts_idx_t *idx = sam_index_load(data[i]->fp, argv[optind+i]); // load the index
+ hts_idx_t *idx = NULL;
+ // If index filename has not been specfied, look in BAM folder
+ if (has_index_file) {
+ idx = sam_index_load2(data[i]->fp, argv[optind+i], argv[optind+i+n]); // load the index
+ } else {
+ idx = sam_index_load(data[i]->fp, argv[optind+i]);
+ }
if (idx == NULL) {
print_error("depth", "can't load index for \"%s\"", argv[optind+i]);
status = EXIT_FAILURE;
goto depth_end;
}
}
+ data[i]->flags = flags;
}
-
+ if (print_header) {
+ fputs("#CHROM\tPOS", file_out);
+ for (i = 0; i < n; ++i) {
+ fputc('\t', file_out);
+ fputs(argv[optind+i], file_out);
+ }
+ fputc('\n', file_out);
+ }
h = data[0]->hdr; // easy access to the header of the 1st BAM
if (reg) {
beg = data[0]->iter->beg; // and to the parsed region coordinates
bam_mplp_set_maxcnt(mplp,INT_MAX);
n_plp = calloc(n, sizeof(int)); // n_plp[i] is the number of covering reads from the i-th BAM
plp = calloc(n, sizeof(bam_pileup1_t*)); // plp[i] points to the array of covering reads (internal in mplp)
- while ((ret=bam_mplp_auto(mplp, &tid, &pos, n_plp, plp)) > 0) { // come to the next covered position
+ while ((ret=bam_mplp64_auto(mplp, &tid, &pos, n_plp, plp)) > 0) { // come to the next covered position
if (pos < beg || pos >= end) continue; // out of range; skip
- if (tid >= h->n_targets) continue; // diff number of @SQ lines per file?
+ if (tid >= sam_hdr_nref(h)) continue; // diff number of @SQ lines per file?
if (all) {
while (tid > last_tid) {
if (last_tid >= 0 && !reg) {
// Deal with remainder or entirety of last tid.
- while (++last_pos < h->target_len[last_tid]) {
+ while (++last_pos < sam_hdr_tid2len(h, last_tid)) {
// Horribly inefficient, but the bed API is an obfuscated black box.
- if (bed && bed_overlap(bed, h->target_name[last_tid], last_pos, last_pos + 1) == 0)
+ if (bed && bed_overlap(bed, sam_hdr_tid2name(h, last_tid), last_pos, last_pos + 1) == 0)
continue;
- fputs(h->target_name[last_tid], stdout); printf("\t%d", last_pos+1);
+ fputs(sam_hdr_tid2name(h, last_tid), file_out);
+ fprintf(file_out, "\t%"PRIhts_pos, last_pos+1);
for (i = 0; i < n; i++)
- putchar('\t'), putchar('0');
- putchar('\n');
+ fputc('\t', file_out), fputc('0', file_out);
+ fputc('\n', file_out);
}
}
last_tid++;
// Deal with missing portion of current tid
while (++last_pos < pos) {
if (last_pos < beg) continue; // out of range; skip
- if (bed && bed_overlap(bed, h->target_name[tid], last_pos, last_pos + 1) == 0)
+ if (bed && bed_overlap(bed, sam_hdr_tid2name(h, tid), last_pos, last_pos + 1) == 0)
continue;
- fputs(h->target_name[tid], stdout); printf("\t%d", last_pos+1);
+ fputs(sam_hdr_tid2name(h, tid), file_out);
+ fprintf(file_out, "\t%"PRIhts_pos, last_pos+1);
for (i = 0; i < n; i++)
- putchar('\t'), putchar('0');
- putchar('\n');
+ fputc('\t', file_out), fputc('0', file_out);
+ fputc('\n', file_out);
}
last_tid = tid;
last_pos = pos;
}
- if (bed && bed_overlap(bed, h->target_name[tid], pos, pos + 1) == 0) continue;
- fputs(h->target_name[tid], stdout); printf("\t%d", pos+1); // a customized printf() would be faster
+ if (bed && bed_overlap(bed, sam_hdr_tid2name(h, tid), pos, pos + 1) == 0) continue;
+ fputs(sam_hdr_tid2name(h, tid), file_out);
+ fprintf(file_out, "\t%"PRIhts_pos, pos+1); // a customized printf() would be faster
for (i = 0; i < n; ++i) { // base level filters have to go here
int j, m = 0;
for (j = 0; j < n_plp[i]; ++j) {
else if (p->qpos < p->b->core.l_qseq &&
bam_get_qual(p->b)[p->qpos] < baseQ) ++m; // low base quality
}
- printf("\t%d", n_plp[i] - m); // this the depth to output
+ fprintf(file_out, "\t%d", n_plp[i] - m); // this the depth to output
}
- putchar('\n');
+ fputc('\n', file_out);
}
if (ret < 0) status = EXIT_FAILURE;
free(n_plp); free(plp);
if (all) {
// Handle terminating region
- if (last_tid < 0 && reg && all > 1) {
+ if (last_tid < 0 && reg) {
last_tid = reg_tid;
last_pos = beg-1;
}
- while (last_tid >= 0 && last_tid < h->n_targets) {
- while (++last_pos < h->target_len[last_tid]) {
+ while (last_tid >= 0 && last_tid < sam_hdr_nref(h)) {
+ while (++last_pos < sam_hdr_tid2len(h, last_tid)) {
if (last_pos >= end) break;
- if (bed && bed_overlap(bed, h->target_name[last_tid], last_pos, last_pos + 1) == 0)
+ if (bed && bed_overlap(bed, sam_hdr_tid2name(h, last_tid), last_pos, last_pos + 1) == 0)
continue;
- fputs(h->target_name[last_tid], stdout); printf("\t%d", last_pos+1);
+ fputs(sam_hdr_tid2name(h, last_tid), file_out);
+ fprintf(file_out, "\t%"PRIhts_pos, last_pos+1);
for (i = 0; i < n; i++)
- putchar('\t'), putchar('0');
- putchar('\n');
+ fputc('\t', file_out), fputc('0', file_out);
+ fputc('\n', file_out);
}
last_tid++;
last_pos = -1;
}
depth_end:
+ if (((file_out != stdout)? fclose(file_out) : fflush(file_out)) != 0) {
+ if (status == EXIT_SUCCESS) {
+ if (file_out != stdout)
+ print_error_errno("depth", "error on closing \"%s\"", output_file);
+ else
+ print_error_errno("depth", "error on flushing standard output");
+ status = EXIT_FAILURE;
+ }
+ }
+
for (i = 0; i < n && data[i]; ++i) {
- bam_hdr_destroy(data[i]->hdr);
+ sam_hdr_destroy(data[i]->hdr);
if (data[i]->fp) sam_close(data[i]->fp);
hts_itr_destroy(data[i]->iter);
free(data[i]);
/* bam2depth.c -- depth subcommand.
Copyright (C) 2011, 2012 Broad Institute.
- Copyright (C) 2012-2014 Genome Research Ltd.
+ Copyright (C) 2012-2016, 2018, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
#include <unistd.h>
#include "htslib/sam.h"
#include "samtools.h"
+#include "bedidx.h"
#include "sam_opts.h"
+#define BAM_FMAX ((BAM_FSUPPLEMENTARY << 1) - 1)
+
typedef struct { // auxiliary data structure
samFile *fp; // the file handle
- bam_hdr_t *hdr; // the file header
+ sam_hdr_t *hdr; // the file header
hts_itr_t *iter; // NULL if a region not specified
int min_mapQ, min_len; // mapQ filter; length filter
+ uint32_t flags; // read filtering flags
} aux_t;
-void *bed_read(const char *fn); // read a BED or position list file
-void bed_destroy(void *_h); // destroy the BED data structure
-int bed_overlap(const void *_h, const char *chr, int beg, int end); // test if chr:beg-end overlaps
-int bed_query(const void *_h, const char *chr, int pos, int *beg, int *end);
-
// This function reads a BAM alignment from one BAM file.
static int read_bam(void *data, bam1_t *b) // read level filters better go here to avoid pileup
{
{
ret = aux->iter? sam_itr_next(aux->fp, aux->iter, b) : sam_read1(aux->fp, aux->hdr, b);
if ( ret<0 ) break;
- if ( b->core.flag & (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP) ) continue;
+ if ( b->core.flag & aux->flags) continue;
if ( (int)b->core.qual < aux->min_mapQ ) continue;
if ( aux->min_len && bam_cigar2qlen(b->core.n_cigar, bam_get_cigar(b)) < aux->min_len ) continue;
break;
fprintf(samtools_stderr, " -a output all positions (including zero depth)\n");
fprintf(samtools_stderr, " -a -a (or -aa) output absolutely all positions, including unused ref. sequences\n");
fprintf(samtools_stderr, " -b <bed> list of positions or regions\n");
+ fprintf(samtools_stderr, " -X use customized index files\n");
fprintf(samtools_stderr, " -f <list> list of input BAM filenames, one per line [null]\n");
+ fprintf(samtools_stderr, " -H print a file header\n");
fprintf(samtools_stderr, " -l <int> read length threshold (ignore reads shorter than <int>) [0]\n");
fprintf(samtools_stderr, " -d/-m <int> maximum coverage depth [8000]. If 0, depth is set to the maximum\n"
" integer value, effectively removing any depth limit.\n"); // the htslib's default
+ fprintf(samtools_stderr, " -o FILE where to write output to [samtools_stdout]\n");
fprintf(samtools_stderr, " -q <int> base quality threshold [0]\n");
fprintf(samtools_stderr, " -Q <int> mapping quality threshold [0]\n");
fprintf(samtools_stderr, " -r <chr:from-to> region\n");
+ fprintf(samtools_stderr, " -g <flags> include reads that have any of the specified flags set [0]\n");
+ fprintf(samtools_stderr, " -G <flags> filter out reads that have any of the specified flags set"
+ " [UNMAP,SECONDARY,QCFAIL,DUP]\n");
- sam_global_opt_help(samtools_stderr, "-.--.-");
+ sam_global_opt_help(samtools_stderr, "-.--.--.");
fprintf(samtools_stderr, "\n");
fprintf(samtools_stderr, "The output is a simple tab-separated table with three columns: reference name,\n");
fprintf(samtools_stderr, "omitted by default; see the -a option.\n");
fprintf(samtools_stderr, "\n");
- return 1;
+ return EXIT_FAILURE;
}
int main_depth(int argc, char *argv[])
{
- int i, n, tid, reg_tid, beg, end, pos, *n_plp, baseQ = 0, mapQ = 0, min_len = 0;
+ int i, n, tid, reg_tid, *n_plp, baseQ = 0, mapQ = 0, min_len = 0, has_index_file = 0;
+ hts_pos_t beg, end, pos, last_pos = -1;
int all = 0, status = EXIT_SUCCESS, nfiles, max_depth = -1;
const bam_pileup1_t **plp;
char *reg = 0; // specified region
void *bed = 0; // BED data structure
char *file_list = NULL, **fn = NULL;
- bam_hdr_t *h = NULL; // BAM header of the 1st input
+ sam_hdr_t *h = NULL; // BAM header of the 1st input
aux_t **data;
bam_mplp_t mplp;
- int last_pos = -1, last_tid = -1, ret;
+ int last_tid = -1, ret;
+ int print_header = 0;
+ char *output_file = NULL;
+ FILE *file_out = samtools_stdout;
+ uint32_t flags = (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP);
+ int tflags = 0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
};
// parse the command line
- while ((n = getopt_long(argc, argv, "r:b:q:Q:l:f:am:d:", lopts, NULL)) >= 0) {
+ while ((n = getopt_long(argc, argv, "r:b:Xq:Q:l:f:am:d:Ho:g:G:", lopts, NULL)) >= 0) {
switch (n) {
case 'l': min_len = atoi(optarg); break; // minimum query length
case 'r': reg = strdup(optarg); break; // parsing a region requires a BAM header
case 'b':
bed = bed_read(optarg); // BED or position list file can be parsed now
- if (!bed) { print_error_errno("depth", "Could not read file \"%s\"", optarg); return 1; }
+ if (!bed) {
+ print_error_errno("depth", "Could not read file \"%s\"", optarg);
+ return EXIT_FAILURE;
+ }
break;
+ case 'X': has_index_file = 1; break;
case 'q': baseQ = atoi(optarg); break; // base quality threshold
case 'Q': mapQ = atoi(optarg); break; // mapping quality threshold
case 'f': file_list = optarg; break;
case 'a': all++; break;
case 'd': case 'm': max_depth = atoi(optarg); break; // maximum coverage depth
+ case 'H': print_header = 1; break;
+ case 'o': output_file = optarg; break;
+ case 'g':
+ tflags = bam_str2flag(optarg);
+ if (tflags < 0 || tflags > BAM_FMAX) {
+ print_error_errno("depth", "Flag value \"%s\" is not supported", optarg);
+ return 1;
+ }
+ flags &= ~tflags;
+ break;
+ case 'G':
+ tflags = bam_str2flag(optarg);
+ if (tflags < 0 || tflags > BAM_FMAX) {
+ print_error_errno("depth", "Flag value \"%s\" is not supported", optarg);
+ return 1;
+ }
+ flags |= tflags;
+ break;
default: if (parse_sam_global_opt(n, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': return usage();
if (optind == argc && !file_list)
return usage();
+ /* output file provided by user */
+ if (output_file != NULL && strcmp(output_file,"-")!=0) {
+ file_out = fopen( output_file, "w" );
+ if (file_out == NULL) {
+ print_error_errno("depth", "Cannot open \"%s\" for writing.", output_file);
+ return EXIT_FAILURE;
+ }
+ }
+
+
// initialize the auxiliary data structures
if (file_list)
{
- if ( read_file_list(file_list,&nfiles,&fn) ) return 1;
+ if (has_index_file) {
+ print_error("depth", "The -f option cannot be combined with -X");
+ return 1;
+ }
+ if ( read_file_list(file_list,&nfiles,&fn) ) return EXIT_FAILURE;
n = nfiles;
argv = fn;
optind = 0;
}
- else
- n = argc - optind; // the number of BAMs on the command line
+ else if (has_index_file) { // Calculate # of input BAM files
+ if ((argc - optind) % 2 != 0) {
+ fprintf(samtools_stderr, "Error: Odd number of filenames detected! Each BAM file should have an index file\n");
+ return 1;
+ }
+ n = (argc - optind) / 2;
+ } else {
+ n = argc - optind;
+ }
data = calloc(n, sizeof(aux_t*)); // data[i] for the i-th input
- reg_tid = 0; beg = 0; end = INT_MAX; // set the default region
+ reg_tid = 0; beg = 0; end = HTS_POS_MAX; // set the default region
+
for (i = 0; i < n; ++i) {
int rf;
data[i] = calloc(1, sizeof(aux_t));
rf = SAM_FLAG | SAM_RNAME | SAM_POS | SAM_MAPQ | SAM_CIGAR | SAM_SEQ;
if (baseQ) rf |= SAM_QUAL;
if (hts_set_opt(data[i]->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
- fprintf(samtools_stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
- return 1;
+ print_error_errno("depth", "Failed to set CRAM_OPT_REQUIRED_FIELDS value");
+ status = EXIT_FAILURE;
+ goto depth_end;
}
if (hts_set_opt(data[i]->fp, CRAM_OPT_DECODE_MD, 0)) {
- fprintf(samtools_stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
- return 1;
+ print_error_errno("depth", "Failed to set CRAM_OPT_DECODE_MD value");
+ status = EXIT_FAILURE;
+ goto depth_end;
}
data[i]->min_mapQ = mapQ; // set the mapQ filter
data[i]->min_len = min_len; // set the qlen filter
data[i]->hdr = sam_hdr_read(data[i]->fp); // read the BAM header
if (data[i]->hdr == NULL) {
- fprintf(samtools_stderr, "Couldn't read header for \"%s\"\n",
- argv[optind+i]);
+ print_error_errno("depth", "Couldn't read header for \"%s\"",
+ argv[optind+i]);
status = EXIT_FAILURE;
goto depth_end;
}
if (reg) { // if a region is specified
- hts_idx_t *idx = sam_index_load(data[i]->fp, argv[optind+i]); // load the index
+ hts_idx_t *idx = NULL;
+ // If index filename has not been specfied, look in BAM folder
+ if (has_index_file) {
+ idx = sam_index_load2(data[i]->fp, argv[optind+i], argv[optind+i+n]); // load the index
+ } else {
+ idx = sam_index_load(data[i]->fp, argv[optind+i]);
+ }
if (idx == NULL) {
print_error("depth", "can't load index for \"%s\"", argv[optind+i]);
status = EXIT_FAILURE;
goto depth_end;
}
}
+ data[i]->flags = flags;
}
-
+ if (print_header) {
+ fputs("#CHROM\tPOS", file_out);
+ for (i = 0; i < n; ++i) {
+ fputc('\t', file_out);
+ fputs(argv[optind+i], file_out);
+ }
+ fputc('\n', file_out);
+ }
h = data[0]->hdr; // easy access to the header of the 1st BAM
if (reg) {
beg = data[0]->iter->beg; // and to the parsed region coordinates
bam_mplp_set_maxcnt(mplp,INT_MAX);
n_plp = calloc(n, sizeof(int)); // n_plp[i] is the number of covering reads from the i-th BAM
plp = calloc(n, sizeof(bam_pileup1_t*)); // plp[i] points to the array of covering reads (internal in mplp)
- while ((ret=bam_mplp_auto(mplp, &tid, &pos, n_plp, plp)) > 0) { // come to the next covered position
+ while ((ret=bam_mplp64_auto(mplp, &tid, &pos, n_plp, plp)) > 0) { // come to the next covered position
if (pos < beg || pos >= end) continue; // out of range; skip
- if (tid >= h->n_targets) continue; // diff number of @SQ lines per file?
+ if (tid >= sam_hdr_nref(h)) continue; // diff number of @SQ lines per file?
if (all) {
while (tid > last_tid) {
if (last_tid >= 0 && !reg) {
// Deal with remainder or entirety of last tid.
- while (++last_pos < h->target_len[last_tid]) {
+ while (++last_pos < sam_hdr_tid2len(h, last_tid)) {
// Horribly inefficient, but the bed API is an obfuscated black box.
- if (bed && bed_overlap(bed, h->target_name[last_tid], last_pos, last_pos + 1) == 0)
+ if (bed && bed_overlap(bed, sam_hdr_tid2name(h, last_tid), last_pos, last_pos + 1) == 0)
continue;
- fputs(h->target_name[last_tid], samtools_stdout); fprintf(samtools_stdout, "\t%d", last_pos+1);
+ fputs(sam_hdr_tid2name(h, last_tid), file_out);
+ fprintf(file_out, "\t%"PRIhts_pos, last_pos+1);
for (i = 0; i < n; i++)
- fputc('\t', samtools_stdout), fputc('0', samtools_stdout);
- fputc('\n', samtools_stdout);
+ fputc('\t', file_out), fputc('0', file_out);
+ fputc('\n', file_out);
}
}
last_tid++;
// Deal with missing portion of current tid
while (++last_pos < pos) {
if (last_pos < beg) continue; // out of range; skip
- if (bed && bed_overlap(bed, h->target_name[tid], last_pos, last_pos + 1) == 0)
+ if (bed && bed_overlap(bed, sam_hdr_tid2name(h, tid), last_pos, last_pos + 1) == 0)
continue;
- fputs(h->target_name[tid], samtools_stdout); fprintf(samtools_stdout, "\t%d", last_pos+1);
+ fputs(sam_hdr_tid2name(h, tid), file_out);
+ fprintf(file_out, "\t%"PRIhts_pos, last_pos+1);
for (i = 0; i < n; i++)
- fputc('\t', samtools_stdout), fputc('0', samtools_stdout);
- fputc('\n', samtools_stdout);
+ fputc('\t', file_out), fputc('0', file_out);
+ fputc('\n', file_out);
}
last_tid = tid;
last_pos = pos;
}
- if (bed && bed_overlap(bed, h->target_name[tid], pos, pos + 1) == 0) continue;
- fputs(h->target_name[tid], samtools_stdout); fprintf(samtools_stdout, "\t%d", pos+1); // a customized fprintf(samtools_stdout, ) would be faster
+ if (bed && bed_overlap(bed, sam_hdr_tid2name(h, tid), pos, pos + 1) == 0) continue;
+ fputs(sam_hdr_tid2name(h, tid), file_out);
+ fprintf(file_out, "\t%"PRIhts_pos, pos+1); // a customized fprintf(samtools_stdout, ) would be faster
for (i = 0; i < n; ++i) { // base level filters have to go here
int j, m = 0;
for (j = 0; j < n_plp[i]; ++j) {
else if (p->qpos < p->b->core.l_qseq &&
bam_get_qual(p->b)[p->qpos] < baseQ) ++m; // low base quality
}
- fprintf(samtools_stdout, "\t%d", n_plp[i] - m); // this the depth to output
+ fprintf(file_out, "\t%d", n_plp[i] - m); // this the depth to output
}
- fputc('\n', samtools_stdout);
+ fputc('\n', file_out);
}
if (ret < 0) status = EXIT_FAILURE;
free(n_plp); free(plp);
if (all) {
// Handle terminating region
- if (last_tid < 0 && reg && all > 1) {
+ if (last_tid < 0 && reg) {
last_tid = reg_tid;
last_pos = beg-1;
}
- while (last_tid >= 0 && last_tid < h->n_targets) {
- while (++last_pos < h->target_len[last_tid]) {
+ while (last_tid >= 0 && last_tid < sam_hdr_nref(h)) {
+ while (++last_pos < sam_hdr_tid2len(h, last_tid)) {
if (last_pos >= end) break;
- if (bed && bed_overlap(bed, h->target_name[last_tid], last_pos, last_pos + 1) == 0)
+ if (bed && bed_overlap(bed, sam_hdr_tid2name(h, last_tid), last_pos, last_pos + 1) == 0)
continue;
- fputs(h->target_name[last_tid], samtools_stdout); fprintf(samtools_stdout, "\t%d", last_pos+1);
+ fputs(sam_hdr_tid2name(h, last_tid), file_out);
+ fprintf(file_out, "\t%"PRIhts_pos, last_pos+1);
for (i = 0; i < n; i++)
- fputc('\t', samtools_stdout), fputc('0', samtools_stdout);
- fputc('\n', samtools_stdout);
+ fputc('\t', file_out), fputc('0', file_out);
+ fputc('\n', file_out);
}
last_tid++;
last_pos = -1;
}
depth_end:
+ if (((file_out != samtools_stdout)? fclose(file_out) : fflush(file_out)) != 0) {
+ if (status == EXIT_SUCCESS) {
+ if (file_out != samtools_stdout)
+ print_error_errno("depth", "error on closing \"%s\"", output_file);
+ else
+ print_error_errno("depth", "error on flushing standard output");
+ status = EXIT_FAILURE;
+ }
+ }
+
for (i = 0; i < n && data[i]; ++i) {
- bam_hdr_destroy(data[i]->hdr);
+ sam_hdr_destroy(data[i]->hdr);
if (data[i]->fp) sam_close(data[i]->fp);
hts_itr_destroy(data[i]->iter);
free(data[i]);
/* bam_addrprg.c -- samtools command to add or replace readgroups.
- Copyright (c) 2013, 2015, 2016 Genome Research Limited.
+ Copyright (c) 2013, 2015-2017, 2019 Genome Research Limited.
Author: Martin O. Pollard <mp15@sanger.ac.uk>
char* output_name;
char* rg_id;
char* rg_line;
+ int no_pg;
rg_mode mode;
sam_global_args ga;
htsThreadPool p;
struct state {
samFile* input_file;
- bam_hdr_t* input_header;
+ sam_hdr_t* input_header;
samFile* output_file;
- bam_hdr_t* output_header;
+ sam_hdr_t* output_header;
char* rg_id;
void (*mode_func)(const state_t*, bam1_t*);
};
free(opts->rg_id);
free(opts->output_name);
free(opts->input_name);
+ free(opts->rg_line);
if (opts->p.pool) hts_tpool_destroy(opts->p.pool);
sam_global_args_free(&opts->ga);
free(opts);
if (!state) return;
free(state->rg_id);
if (state->output_file) sam_close(state->output_file);
- bam_hdr_destroy(state->output_header);
+ sam_hdr_destroy(state->output_header);
if (state->input_file) sam_close(state->input_file);
- bam_hdr_destroy(state->input_header);
+ sam_hdr_destroy(state->input_header);
free(state);
}
return ns;
}
-// These are to be replaced by samtools header parser
-// Extracts the first @RG line from a string.
-static char* get_rg_line(const char* text, size_t* last)
-{
- const char* rg = text;
- if (rg[0] != '@' || rg[1] != 'R' || rg[2] != 'G' ) {
- if ((rg = (const char*)strstr(text,"\n@RG")) == NULL) {
- return NULL;
- }
- rg++;//skip initial \n
- }
- // duplicate the line for return
- return dup_substring(rg, strchr(rg, '\n'), last);
-}
// Given a @RG line return the id
static char* get_rg_id(const char *line)
return dup_substring(id, strchr(id, '\t'), NULL);
}
-// Confirms the existance of an RG line with a given ID in a bam header
-static bool confirm_rg( const bam_hdr_t *hdr, const char* rgid )
-{
- assert( hdr != NULL && rgid != NULL );
-
- const char *ptr = hdr->text;
- bool found = false;
- while (ptr != NULL && *ptr != '\0' && found == false ) {
- size_t end = 0;
- char* line = get_rg_line(ptr, &end);
- if (line == NULL) break; // No more @RG
- char* id;
- if (((id = get_rg_id(line)) != NULL) && !strcmp(id, rgid)) {
- found = true;
- }
- free(id);
- free(line);
- ptr += end;
- }
- return found;
-}
-
-static char* get_first_rgid( const bam_hdr_t *hdr )
-{
- assert( hdr != NULL );
- const char *ptr = hdr->text;
- char* found = NULL;
- while (ptr != NULL && *ptr != '\0' && found == NULL ) {
- size_t end = 0;
- char* line = get_rg_line(ptr, &end);
- if ( line ) {
- found = get_rg_id(line);
- } else break;
- free(line);
- ptr += end;
- }
- return found;
-}
static void usage(FILE *fp)
{
" -o FILE Where to write output to [stdout]\n"
" -r STRING @RG line text\n"
" -R STRING ID of @RG line in existing header to use\n"
+ " --no-PG Do not add a PG line\n"
);
- sam_global_opt_help(fp, "..O..@");
+ sam_global_opt_help(fp, "..O..@..");
}
static bool parse_args(int argc, char** argv, parsed_opts_t** opts)
sam_global_args_init(&retval->ga);
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS(0, 0, 'O', 0, 0, '@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
kstring_t rg_line = {0,0,NULL};
usage(stdout);
free(retval);
return true;
+ case 1:
+ retval->no_pg = 1;
+ break;
case '?':
usage(stderr);
free(retval);
cleanup_opts(retval);
return false;
}
+ free(retval->rg_line);
retval->rg_line = tmp;
}
retval->input_name = strdup(argv[optind+0]);
}
retval->input_header = sam_hdr_read(retval->input_file);
- retval->output_header = bam_hdr_dup(retval->input_header);
+ retval->output_header = sam_hdr_dup(retval->input_header);
if (opts->output_name) // File format auto-detection
sam_open_mode(output_mode + 1, opts->output_name, NULL);
retval->output_file = sam_open_format(opts->output_name == NULL?"-":opts->output_name, output_mode, &opts->ga.out);
if (opts->rg_line) {
// Append new RG line to header.
// Check does not already exist
- if ( confirm_rg(retval->output_header, opts->rg_id) ) {
+ kstring_t hdr_line = { 0, 0, NULL };
+ if (sam_hdr_find_line_id(retval->output_header, "RG", "ID", opts->rg_id, &hdr_line) == 0) {
fprintf(stderr, "[init] ID of new RG line specified conflicts with that of an existing header RG line. Overwrite not yet implemented.\n");
+ free(hdr_line.s);
return false;
}
- retval->rg_id = strdup(opts->rg_id);
- size_t new_len = strlen( retval->output_header->text ) + strlen( opts->rg_line ) + 2;
- char* new_header = malloc(new_len);
- if (!new_header) {
- fprintf(stderr, "[init] Out of memory whilst writing new header.\n");
+ if (-1 == sam_hdr_add_lines(retval->output_header, opts->rg_line, strlen(opts->rg_line))) {
+ fprintf(stderr, "[init] Error adding RG line with ID:%s to the output header.\n", opts->rg_id);
return false;
}
- sprintf(new_header,"%s%s\n", retval->output_header->text, opts->rg_line);
- free(retval->output_header->text);
- retval->output_header->text = new_header;
- retval->output_header->l_text = (int)new_len - 1;
+ if (opts->mode == overwrite_all &&
+ -1 == sam_hdr_remove_except(retval->output_header, "RG", "ID", opts->rg_id)) {
+ fprintf(stderr, "[init] Error removing the old RG lines from the output header.\n");
+ return false;
+ }
+ retval->rg_id = strdup(opts->rg_id);
} else {
if (opts->rg_id) {
// Confirm what has been supplied exists
- if ( !confirm_rg(retval->output_header, opts->rg_id) ) {
+ kstring_t hdr_line = { 0, 0, NULL };
+ if (sam_hdr_find_line_id(retval->output_header, "RG", "ID", opts->rg_id, &hdr_line) < 0) {
fprintf(stderr, "RG ID supplied does not exist in header. Supply full @RG line with -r instead?\n");
return false;
}
retval->rg_id = strdup(opts->rg_id);
+ free(hdr_line.s);
} else {
- if ((retval->rg_id = get_first_rgid(retval->output_header)) == NULL ) {
+ kstring_t rg_id = { 0, 0, NULL };
+ if (sam_hdr_find_tag_id(retval->output_header, "RG", NULL, NULL, "ID", &rg_id) < 0) {
fprintf(stderr, "No RG specified on command line or in existing header.\n");
return false;
}
+ retval->rg_id = ks_release(&rg_id);
}
}
return true;
}
-static bool readgroupise(state_t* state)
+static bool readgroupise(parsed_opts_t *opts, state_t* state, char *arg_list)
{
+ if (!opts->no_pg && sam_hdr_add_pg(state->output_header, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))
+ return false;
+
if (sam_hdr_write(state->output_file, state->output_header) != 0) {
print_error_errno("addreplacerg", "[%s] Could not write header to output file", __func__);
return false;
}
+ char *idx_fn = NULL;
+ if (opts->ga.write_index) {
+ if (!(idx_fn = auto_index(state->output_file, opts->output_name, state->output_header)))
+ return false;
+ }
bam1_t* file_read = bam_init1();
int ret;
if (sam_write1(state->output_file, state->output_header, file_read) < 0) {
print_error_errno("addreplacerg", "[%s] Could not write read to output file", __func__);
bam_destroy1(file_read);
+ free(idx_fn);
return false;
}
}
bam_destroy1(file_read);
if (ret != -1) {
print_error_errno("addreplacerg", "[%s] Error reading from input file", __func__);
+ free(idx_fn);
return false;
} else {
+
+ if (opts->ga.write_index) {
+ if (sam_idx_save(state->output_file) < 0) {
+ print_error_errno("addreplacerg", "[%s] Writing index failed", __func__);
+ free(idx_fn);
+ return false;
+ }
+ }
+ free(idx_fn);
return true;
}
}
{
parsed_opts_t* opts = NULL;
state_t* state = NULL;
+ char *arg_list = stringify_argv(argc+1, argv-1);
+ if (!arg_list)
+ return EXIT_FAILURE;
if (!parse_args(argc, argv, &opts)) goto error;
- if (opts == NULL) return EXIT_SUCCESS; // Not an error but user doesn't want us to proceed
- if (!opts || !init(opts, &state)) goto error;
-
- if (!readgroupise(state)) goto error;
+ if (opts) { // Not an error but user doesn't want us to proceed
+ if (!init(opts, &state) || !readgroupise(opts, state, arg_list))
+ goto error;
+ }
cleanup_state(state);
cleanup_opts(opts);
+ free(arg_list);
return EXIT_SUCCESS;
error:
cleanup_state(state);
cleanup_opts(opts);
+ free(arg_list);
return EXIT_FAILURE;
}
/* bam_addrprg.c -- samtools command to add or replace readgroups.
- Copyright (c) 2013, 2015, 2016 Genome Research Limited.
+ Copyright (c) 2013, 2015-2017, 2019 Genome Research Limited.
Author: Martin O. Pollard <mp15@sanger.ac.uk>
char* output_name;
char* rg_id;
char* rg_line;
+ int no_pg;
rg_mode mode;
sam_global_args ga;
htsThreadPool p;
struct state {
samFile* input_file;
- bam_hdr_t* input_header;
+ sam_hdr_t* input_header;
samFile* output_file;
- bam_hdr_t* output_header;
+ sam_hdr_t* output_header;
char* rg_id;
void (*mode_func)(const state_t*, bam1_t*);
};
free(opts->rg_id);
free(opts->output_name);
free(opts->input_name);
+ free(opts->rg_line);
if (opts->p.pool) hts_tpool_destroy(opts->p.pool);
sam_global_args_free(&opts->ga);
free(opts);
if (!state) return;
free(state->rg_id);
if (state->output_file) sam_close(state->output_file);
- bam_hdr_destroy(state->output_header);
+ sam_hdr_destroy(state->output_header);
if (state->input_file) sam_close(state->input_file);
- bam_hdr_destroy(state->input_header);
+ sam_hdr_destroy(state->input_header);
free(state);
}
return ns;
}
-// These are to be replaced by samtools header parser
-// Extracts the first @RG line from a string.
-static char* get_rg_line(const char* text, size_t* last)
-{
- const char* rg = text;
- if (rg[0] != '@' || rg[1] != 'R' || rg[2] != 'G' ) {
- if ((rg = (const char*)strstr(text,"\n@RG")) == NULL) {
- return NULL;
- }
- rg++;//skip initial \n
- }
- // duplicate the line for return
- return dup_substring(rg, strchr(rg, '\n'), last);
-}
// Given a @RG line return the id
static char* get_rg_id(const char *line)
return dup_substring(id, strchr(id, '\t'), NULL);
}
-// Confirms the existance of an RG line with a given ID in a bam header
-static bool confirm_rg( const bam_hdr_t *hdr, const char* rgid )
-{
- assert( hdr != NULL && rgid != NULL );
-
- const char *ptr = hdr->text;
- bool found = false;
- while (ptr != NULL && *ptr != '\0' && found == false ) {
- size_t end = 0;
- char* line = get_rg_line(ptr, &end);
- if (line == NULL) break; // No more @RG
- char* id;
- if (((id = get_rg_id(line)) != NULL) && !strcmp(id, rgid)) {
- found = true;
- }
- free(id);
- free(line);
- ptr += end;
- }
- return found;
-}
-
-static char* get_first_rgid( const bam_hdr_t *hdr )
-{
- assert( hdr != NULL );
- const char *ptr = hdr->text;
- char* found = NULL;
- while (ptr != NULL && *ptr != '\0' && found == NULL ) {
- size_t end = 0;
- char* line = get_rg_line(ptr, &end);
- if ( line ) {
- found = get_rg_id(line);
- } else break;
- free(line);
- ptr += end;
- }
- return found;
-}
static void usage(FILE *fp)
{
" -o FILE Where to write output to [samtools_stdout]\n"
" -r STRING @RG line text\n"
" -R STRING ID of @RG line in existing header to use\n"
+ " --no-PG Do not add a PG line\n"
);
- sam_global_opt_help(fp, "..O..@");
+ sam_global_opt_help(fp, "..O..@..");
}
static bool parse_args(int argc, char** argv, parsed_opts_t** opts)
sam_global_args_init(&retval->ga);
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS(0, 0, 'O', 0, 0, '@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
kstring_t rg_line = {0,0,NULL};
usage(samtools_stdout);
free(retval);
return true;
+ case 1:
+ retval->no_pg = 1;
+ break;
case '?':
usage(samtools_stderr);
free(retval);
cleanup_opts(retval);
return false;
}
+ free(retval->rg_line);
retval->rg_line = tmp;
}
retval->input_name = strdup(argv[optind+0]);
}
retval->input_header = sam_hdr_read(retval->input_file);
- retval->output_header = bam_hdr_dup(retval->input_header);
+ retval->output_header = sam_hdr_dup(retval->input_header);
if (opts->output_name) // File format auto-detection
sam_open_mode(output_mode + 1, opts->output_name, NULL);
retval->output_file = sam_open_format(opts->output_name == NULL?"-":opts->output_name, output_mode, &opts->ga.out);
if (opts->rg_line) {
// Append new RG line to header.
// Check does not already exist
- if ( confirm_rg(retval->output_header, opts->rg_id) ) {
+ kstring_t hdr_line = { 0, 0, NULL };
+ if (sam_hdr_find_line_id(retval->output_header, "RG", "ID", opts->rg_id, &hdr_line) == 0) {
fprintf(samtools_stderr, "[init] ID of new RG line specified conflicts with that of an existing header RG line. Overwrite not yet implemented.\n");
+ free(hdr_line.s);
return false;
}
- retval->rg_id = strdup(opts->rg_id);
- size_t new_len = strlen( retval->output_header->text ) + strlen( opts->rg_line ) + 2;
- char* new_header = malloc(new_len);
- if (!new_header) {
- fprintf(samtools_stderr, "[init] Out of memory whilst writing new header.\n");
+ if (-1 == sam_hdr_add_lines(retval->output_header, opts->rg_line, strlen(opts->rg_line))) {
+ fprintf(samtools_stderr, "[init] Error adding RG line with ID:%s to the output header.\n", opts->rg_id);
return false;
}
- sprintf(new_header,"%s%s\n", retval->output_header->text, opts->rg_line);
- free(retval->output_header->text);
- retval->output_header->text = new_header;
- retval->output_header->l_text = (int)new_len - 1;
+ if (opts->mode == overwrite_all &&
+ -1 == sam_hdr_remove_except(retval->output_header, "RG", "ID", opts->rg_id)) {
+ fprintf(samtools_stderr, "[init] Error removing the old RG lines from the output header.\n");
+ return false;
+ }
+ retval->rg_id = strdup(opts->rg_id);
} else {
if (opts->rg_id) {
// Confirm what has been supplied exists
- if ( !confirm_rg(retval->output_header, opts->rg_id) ) {
+ kstring_t hdr_line = { 0, 0, NULL };
+ if (sam_hdr_find_line_id(retval->output_header, "RG", "ID", opts->rg_id, &hdr_line) < 0) {
fprintf(samtools_stderr, "RG ID supplied does not exist in header. Supply full @RG line with -r instead?\n");
return false;
}
retval->rg_id = strdup(opts->rg_id);
+ free(hdr_line.s);
} else {
- if ((retval->rg_id = get_first_rgid(retval->output_header)) == NULL ) {
+ kstring_t rg_id = { 0, 0, NULL };
+ if (sam_hdr_find_tag_id(retval->output_header, "RG", NULL, NULL, "ID", &rg_id) < 0) {
fprintf(samtools_stderr, "No RG specified on command line or in existing header.\n");
return false;
}
+ retval->rg_id = ks_release(&rg_id);
}
}
return true;
}
-static bool readgroupise(state_t* state)
+static bool readgroupise(parsed_opts_t *opts, state_t* state, char *arg_list)
{
+ if (!opts->no_pg && sam_hdr_add_pg(state->output_header, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))
+ return false;
+
if (sam_hdr_write(state->output_file, state->output_header) != 0) {
print_error_errno("addreplacerg", "[%s] Could not write header to output file", __func__);
return false;
}
+ char *idx_fn = NULL;
+ if (opts->ga.write_index) {
+ if (!(idx_fn = auto_index(state->output_file, opts->output_name, state->output_header)))
+ return false;
+ }
bam1_t* file_read = bam_init1();
int ret;
if (sam_write1(state->output_file, state->output_header, file_read) < 0) {
print_error_errno("addreplacerg", "[%s] Could not write read to output file", __func__);
bam_destroy1(file_read);
+ free(idx_fn);
return false;
}
}
bam_destroy1(file_read);
if (ret != -1) {
print_error_errno("addreplacerg", "[%s] Error reading from input file", __func__);
+ free(idx_fn);
return false;
} else {
+
+ if (opts->ga.write_index) {
+ if (sam_idx_save(state->output_file) < 0) {
+ print_error_errno("addreplacerg", "[%s] Writing index failed", __func__);
+ free(idx_fn);
+ return false;
+ }
+ }
+ free(idx_fn);
return true;
}
}
{
parsed_opts_t* opts = NULL;
state_t* state = NULL;
+ char *arg_list = stringify_argv(argc+1, argv-1);
+ if (!arg_list)
+ return EXIT_FAILURE;
if (!parse_args(argc, argv, &opts)) goto error;
- if (opts == NULL) return EXIT_SUCCESS; // Not an error but user doesn't want us to proceed
- if (!opts || !init(opts, &state)) goto error;
-
- if (!readgroupise(state)) goto error;
+ if (opts) { // Not an error but user doesn't want us to proceed
+ if (!init(opts, &state) || !readgroupise(opts, state, arg_list))
+ goto error;
+ }
cleanup_state(state);
cleanup_opts(opts);
+ free(arg_list);
return EXIT_SUCCESS;
error:
cleanup_state(state);
cleanup_opts(opts);
+ free(arg_list);
return EXIT_FAILURE;
}
/* bam_aux.c -- remaining aux field handling.
- Copyright (C) 2008-2010, 2013 Genome Research Ltd.
+ Copyright (C) 2008-2010, 2013, 2015, 2019 Genome Research Ltd.
Portions copyright (C) 2011 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
return 0;
}
+// Only here due to libbam.a being used by some applications.
int bam_parse_region(bam_header_t *header, const char *str, int *ref_id, int *beg, int *end)
{
- const char *name_lim = hts_parse_reg(str, beg, end);
- if (name_lim) {
- char *name = malloc(name_lim - str + 1);
- memcpy(name, str, name_lim - str);
- name[name_lim - str] = '\0';
- *ref_id = bam_name2id(header, name);
- free(name);
- }
- else {
- // not parsable as a region, but possibly a sequence named "foo:a"
- *ref_id = bam_name2id(header, str);
- *beg = 0; *end = INT_MAX;
- }
- if (*ref_id == -1) return -1;
- return *beg <= *end? 0 : -1;
+ hts_pos_t beg64, end64;
+ int r;
+ r = sam_parse_region(header, str, ref_id, &beg64, &end64, 0) ? 0 : -1;
+ if (beg64 > INT_MAX || end64 > INT_MAX)
+ return -1;
+ *beg = beg64;
+ *end = end64;
+ return r;
}
/* bam_aux.c -- remaining aux field handling.
- Copyright (C) 2008-2010, 2013 Genome Research Ltd.
+ Copyright (C) 2008-2010, 2013, 2015, 2019 Genome Research Ltd.
Portions copyright (C) 2011 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
return 0;
}
+// Only here due to libbam.a being used by some applications.
int bam_parse_region(bam_header_t *header, const char *str, int *ref_id, int *beg, int *end)
{
- const char *name_lim = hts_parse_reg(str, beg, end);
- if (name_lim) {
- char *name = malloc(name_lim - str + 1);
- memcpy(name, str, name_lim - str);
- name[name_lim - str] = '\0';
- *ref_id = bam_name2id(header, name);
- free(name);
- }
- else {
- // not parsable as a region, but possibly a sequence named "foo:a"
- *ref_id = bam_name2id(header, str);
- *beg = 0; *end = INT_MAX;
- }
- if (*ref_id == -1) return -1;
- return *beg <= *end? 0 : -1;
+ hts_pos_t beg64, end64;
+ int r;
+ r = sam_parse_region(header, str, ref_id, &beg64, &end64, 0) ? 0 : -1;
+ if (beg64 > INT_MAX || end64 > INT_MAX)
+ return -1;
+ *beg = beg64;
+ *end = end64;
+ return r;
}
/* bam_cat.c -- efficiently concatenates bam files.
- Copyright (C) 2008-2009, 2011-2013, 2015-2016 Genome Research Ltd.
+ Copyright (C) 2008-2009, 2011-2013, 2015-2017, 2019 Genome Research Ltd.
Modified SAMtools work copyright (C) 2010 Illumina, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
#include "htslib/bgzf.h"
#include "htslib/sam.h"
#include "htslib/cram.h"
-#include "htslib/khash.h"
+#include "htslib/kstring.h"
#include "samtools.h"
-
-KHASH_MAP_INIT_STR(s2i, int)
-
-// Bi-directional lookup.
-// We can go from name to ID or ID to name.
-typedef struct khash_s2i {
- khash_t(s2i) *h;
- int n_id, a_id;
- const char **id; // map Nth entry back to key
- const char **line;
-} khash_s2i;
-
-static int hash_s2i_inc(khash_s2i *hash, const char *str, const char *line, int *added) {
- // loosly based on khash_str2int_inc
- khint_t k;
- int n;
-
- if ( !hash ) return -1;
- // inefficient, but works
- char *my_str = strdup(str);
- k = kh_put(s2i, hash->h, my_str, added);
- if (*added == 0) {
- free(my_str);
- return kh_val(hash->h, k);
- }
- n = hash->n_id++;
- kh_val(hash->h, k) = n;
- if (hash->a_id <= n) {
- const char **id;
- hash->a_id = (n+1)*2;
- if (!(id = realloc(hash->id, hash->a_id*sizeof(*hash->id))))
- return -1;
- hash->id = id;
- if (!(id = realloc(hash->line, hash->a_id*sizeof(*hash->line))))
- return -1;
- hash->line = id;
- }
- hash->id[n] = my_str; // reverse map
- if (line)
- hash->line[n] = line;
-
- return n;
-}
-
-khash_s2i *hash_s2i_create(void) {
- khash_s2i *h = calloc(1, sizeof(*h));
- if (!h)
- return NULL;
-
- h->h = kh_init(s2i);
- if (!h->h) {
- free(h);
- return NULL;
- }
- return h;
-}
-
-static void hash_s2i_free(khash_s2i *hash) {
- // based on khash_str2int_destroy_free
- khint_t k;
- if (!hash) return;
- if (hash->h) {
- for (k = 0; k < kh_end(hash->h); ++k)
- if (kh_exist(hash->h, k)) free((char*)kh_key(hash->h, k));
- kh_destroy(s2i, hash->h);
- }
- if (hash->id)
- free(hash->id);
- if (hash->line)
- free(hash->line);
-
- free(hash);
-}
-
-static khash_s2i *hash_rg(const bam_hdr_t *h) {
- khash_s2i *rg2id = hash_s2i_create();
- char *cp, *line;
- int j, l;
-
- if (!h)
- return rg2id;
-
- if (!rg2id)
- return NULL;
-
- cp = h->text;
-
- for (l = 0; l+3 < h->l_text; l++) {
- line = &cp[l];
- if (!(cp[l] == '@' && cp[l+1] == 'R' && cp[l+2] == 'G')) {
- while (l < h->l_text && cp[l] != '\n')
- l++;
- continue;
- }
-
- // Found an @RG line; add to hash
- while (cp[l] != '\n') {
- while (l < h->l_text && cp[l] != '\n' && cp[l] != '\t')
- l++;
- if (l+4 < h->l_text && cp[l+1] == 'I' && cp[l+2] == 'D')
- break;
- }
- if (cp[l] == '\n')
- continue;
- l = (j = l+4);
- while (l < h->l_text && cp[l] != '\n' && cp[l] != '\t')
- l++;
-
- // To do: save id and keep realloc as needed, as hash_s2i_inc strdups.
- char *id = malloc(l-j+1);
- strncpy(id, &cp[j], l-j);
- id[l-j] = 0;
-
- int added;
- hash_s2i_inc(rg2id, id, line, &added);
- free(id);
-
- while (l < h->l_text && cp[l] != '\n')
- l++;
- }
-
- return rg2id;
-}
+#include "sam_opts.h"
/*
* Check the files are consistent and capable of being concatenated.
- * Also fills out the rg2id read-group hash and the version numbers
- * and produces a new bam_hdr_t structure with merged RG lines.
- * Note it is only a simple merge, as we lack the niceties of a proper
- * header API.
+ * Also fills out the version numbers and produces a new sam_hdr_t
+ * structure with merged RG lines.
+ * Note it is only a simple merge.
*
* Returns updated header on success;
* NULL on failure.
*/
-static bam_hdr_t *cram_cat_check_hdr(int nfn, char * const *fn, const bam_hdr_t *h,
- khash_s2i **rg2id, int *vers_maj_p, int *vers_min_p) {
+static sam_hdr_t *cram_cat_check_hdr(int nfn, char * const *fn, const sam_hdr_t *h,
+ int *vers_maj_p, int *vers_min_p) {
int i, vers_maj = -1, vers_min = -1;
- bam_hdr_t *new_h = NULL;
+ sam_hdr_t *new_h = NULL, *old_h = NULL;
+ samFile *in = NULL;
+ kstring_t ks = KS_INITIALIZE;
if (h) {
- new_h = bam_hdr_dup(h);
- *rg2id = hash_rg(new_h);
+ new_h = sam_hdr_dup(h);
+ if (!new_h) {
+ fprintf(stderr, "[%s] ERROR: header duplication failed.\n",
+ __func__);
+ goto fail;
+ }
}
for (i = 0; i < nfn; ++i) {
- samFile *in;
cram_fd *in_c;
- khint_t ki;
- int new_rg = -1;
+ int ki;
in = sam_open(fn[i], "rc");
if (in == 0) {
print_error_errno("cat", "fail to open file '%s'", fn[i]);
- return NULL;
+ goto fail;
}
in_c = in->fp.cram;
(vers_min != -1 && vers_min != vmin)) {
fprintf(stderr, "[%s] ERROR: input files have differing version numbers.\n",
__func__);
- return NULL;
+ goto fail;
}
vers_maj = vmaj;
vers_min = vmin;
- bam_hdr_t *old = sam_hdr_read(in);
- khash_s2i *rg2id_in = hash_rg(old);
+ old_h = sam_hdr_read(in);
+ if (!old_h) {
+ fprintf(stderr, "[%s] ERROR: header reading for file '%s' filed.\n",
+ __func__, fn[i]);
+ goto fail;
+ }
if (!new_h) {
- new_h = bam_hdr_dup(old);
- *rg2id = hash_rg(new_h);
+ new_h = sam_hdr_dup(old_h);
+ if (!new_h) {
+ fprintf(stderr, "[%s] ERROR: header duplication for file '%s' failed.\n",
+ __func__, fn[i]);
+ goto fail;
+ }
+ sam_hdr_destroy(old_h);
+ sam_close(in);
+ continue;
}
- // Add any existing @RG entries to our global @RG hash.
- for (ki = 0; ki < rg2id_in->n_id; ki++) {
- int added;
-
- new_rg = hash_s2i_inc(*rg2id, rg2id_in->id[ki], rg2id_in->line[ki], &added);
- //fprintf(stderr, "RG %s: #%d -> #%d\n",
- // rg2id_in->id[ki], ki, new_rg);
-
- if (added) {
- // Also add to new_h
- const char *line = rg2id_in->line[ki];
- const char *line_end = line;
- while (*line && *line_end++ != '\n')
- ;
- new_h->l_text += line_end - line;
- new_h->text = realloc(new_h->text, new_h->l_text+1);
- strncat(&new_h->text[new_h->l_text - (line_end - line)],
- line, line_end - line);
+ int old_count = sam_hdr_count_lines(old_h, "RG");
+ for (ki = 0; ki < old_count; ki++) {
+ const char *old_name = sam_hdr_line_name(old_h, "RG", ki);
+ if (old_name) {
+ int new_i = sam_hdr_line_index(new_h, "RG", old_name);
+ if (-1 == new_i) { // line does not exist in the new header
+ if (sam_hdr_find_line_pos(old_h, "RG", ki, &ks) ||
+ !ks.s || sam_hdr_add_lines(new_h, ks.s, ks.l)) {
+ fprintf(stderr, "[%s] ERROR: failed to add @RG line 'ID:%s' from file '%s'\n",
+ __func__, old_name, fn[i]);
+ goto fail;
+ }
+ ks_free(&ks);
+ }
+ } else {
+ fprintf(stderr, "[%s] ERROR: failed to read %d @RG line from file '%s'\n",
+ __func__, ki, fn[i]);
+ goto fail;
}
+ }
- if (new_rg != ki && rg2id_in->n_id > 1) {
- fprintf(stderr, "[%s] ERROR: Same size @RG lists but differing order / contents\n",
- __func__);
- return NULL;
+ if (old_count > 1 && sam_hdr_count_lines(new_h, "RG") == old_count) {
+ for (ki = 0; ki < old_count; ki++) {
+ const char *old_name = sam_hdr_line_name(old_h, "RG", ki);
+ const char *new_name = sam_hdr_line_name(new_h, "RG", ki);
+ if (!old_name || !new_name || strcmp(old_name, new_name)) {
+ fprintf(stderr, "[%s] ERROR: Same size @RG lists but differing order / contents\n",
+ __func__);
+ goto fail;
+ }
}
}
- hash_s2i_free(rg2id_in);
- bam_hdr_destroy(old);
+ sam_hdr_destroy(old_h);
sam_close(in);
}
+ ks_free(&ks);
+
*vers_maj_p = vers_maj;
*vers_min_p = vers_min;
return new_h;
+
+fail:
+ ks_free(&ks);
+ if (old_h) sam_hdr_destroy(old_h);
+ if (new_h) sam_hdr_destroy(new_h);
+ if (in) sam_close(in);
+
+ return NULL;
}
* huffman code. In this situation we can change the meta-data in the
* compression header to renumber an RG value..
*/
-int cram_cat(int nfn, char * const *fn, const bam_hdr_t *h, const char* outcram)
+int cram_cat(int nfn, char * const *fn, const sam_hdr_t *h, const char* outcram, sam_global_args *ga, char *arg_list, int no_pg)
{
samFile *out;
cram_fd *out_c;
int i, vers_maj, vers_min;
- khash_s2i *rg2id = NULL;
- bam_hdr_t *new_h = NULL;
+ sam_hdr_t *new_h = NULL;
/* Check consistent versioning and compatible headers */
- if (!(new_h = cram_cat_check_hdr(nfn, fn, h, &rg2id, &vers_maj, &vers_min)))
+ if (!(new_h = cram_cat_check_hdr(nfn, fn, h, &vers_maj, &vers_min)))
return -1;
/* Open the file with cram_vers */
char vers[100];
sprintf(vers, "%d.%d", vers_maj, vers_min);
- out = sam_open(outcram, "wc");
+ out = sam_open_format(outcram, "wc", &ga->out);
if (out == 0) {
print_error_errno("cat", "fail to open output file '%s'", outcram);
return -1;
cram_set_option(out_c, CRAM_OPT_VERSION, vers);
//fprintf(stderr, "Creating cram vers %s\n", vers);
- cram_fd_set_header(out_c, sam_hdr_parse_(new_h->text, new_h->l_text)); // needed?
+ if (!no_pg && sam_hdr_add_pg(new_h, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))
+ return -1;
+
if (sam_hdr_write(out, new_h) < 0) {
print_error_errno("cat", "Couldn't write header");
return -1;
samFile *in;
cram_fd *in_c;
cram_container *c;
- bam_hdr_t *old;
+ sam_hdr_t *old_h;
int new_rg = -1;
in = sam_open(fn[i], "rc");
}
in_c = in->fp.cram;
- old = sam_hdr_read(in);
- khash_s2i *rg2id_in = hash_rg(old);
+ old_h = sam_hdr_read(in);
+ if (!old_h) {
+ print_error("cat", "fail to read the header of file '%s'", fn[i]);
+ return -1;
+ }
// Compute RG mapping if suitable for changing.
- if (rg2id_in->n_id == 1) {
- int _;
- new_rg = hash_s2i_inc(rg2id, rg2id_in->id[0], NULL, &_);
+ if (sam_hdr_count_lines(old_h, "RG") == 1) {
+ const char *old_name = sam_hdr_line_name(old_h, "RG", 0);
+ if (old_name) {
+ new_rg = sam_hdr_line_index(new_h, "RG", old_name);
+ if (new_rg < 0) {
+ print_error("cat", "fail to find @RG line '%s' in the new header", old_name);
+ return -1;
+ }
+ } else {
+ print_error("cat", "fail to find @RG line in file '%s'", fn[i]);
+ return -1;
+ }
} else {
new_rg = 0;
}
- hash_s2i_free(rg2id_in);
-
-
// Copy contains and blocks within them
while ((c = cram_read_container(in_c))) {
cram_block *blk;
cram_free_container(c);
}
- bam_hdr_destroy(old);
+ sam_hdr_destroy(old_h);
sam_close(in);
}
sam_close(out);
-
- hash_s2i_free(rg2id);
- bam_hdr_destroy(new_h);
+ sam_hdr_destroy(new_h);
return 0;
}
#define BGZF_EMPTY_BLOCK_SIZE 28
-int bam_cat(int nfn, char * const *fn, const bam_hdr_t *h, const char* outbam)
+int bam_cat(int nfn, char * const *fn, sam_hdr_t *h, const char* outbam, char *arg_list, int no_pg)
{
BGZF *fp, *in = NULL;
uint8_t *buf = NULL;
return -1;
}
if (h) {
+ if (!no_pg && sam_hdr_add_pg(h, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))
+ goto fail;
+
if (bam_hdr_write(fp, h) < 0) {
print_error_errno("cat", "Couldn't write header");
goto fail;
goto fail;
}
for(i = 0; i < nfn; ++i){
- bam_hdr_t *old;
+ sam_hdr_t *old;
int len,j;
in = strcmp(fn[i], "-")? bgzf_open(fn[i], "r") : bgzf_fdopen(fileno(stdin), "r");
goto fail;
}
if (h == 0 && i == 0) {
+ if (!no_pg && sam_hdr_add_pg(old, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))
+ goto fail;
+
if (bam_hdr_write(fp, old) < 0) {
print_error_errno("cat", "Couldn't write header");
goto fail;
if (bgzf_raw_write(fp, ebuf, es) < 0) goto write_fail;
}
}
- bam_hdr_destroy(old);
+ sam_hdr_destroy(old);
bgzf_close(in);
in = NULL;
}
int main_cat(int argc, char *argv[])
{
- bam_hdr_t *h = 0;
+ sam_hdr_t *h = 0;
char *outfn = 0;
char **infns = NULL; // files to concatenate
int infns_size = 0;
- int c, ret = 0;
+ int c, ret = 0, no_pg = 0;
samFile *in;
+ sam_global_args ga;
+
+ static const struct option lopts[] = {
+ SAM_OPT_GLOBAL_OPTIONS('-', '-', '-', 0, '-', '@'),
+ {"no-PG", no_argument, NULL, 1},
+ { NULL, 0, NULL, 0 }
+ };
+
+ char *arg_list = NULL;
- while ((c = getopt(argc, argv, "h:o:b:")) >= 0) {
+ sam_global_args_init(&ga);
+
+ while ((c = getopt_long(argc, argv, "h:o:b:", lopts, NULL)) >= 0) {
switch (c) {
case 'h': {
samFile *fph = sam_open(optarg, "r");
}
break;
}
+ case 1:
+ no_pg = 1;
+ break;
+ default:
+ if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
}
}
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("cat", "failed to create arg_list");
+ return 1;
+ }
+
// Append files specified in argv to the list.
int nargv_fns = argc - optind;
if (nargv_fns > 0) {
fprintf(stderr, "Options: -b FILE list of input BAM/CRAM file names, one per line\n");
fprintf(stderr, " -h FILE copy the header from FILE [default is 1st input file]\n");
fprintf(stderr, " -o FILE output BAM/CRAM\n");
+ fprintf(stderr, " --no-PG do not add a PG line\n");
+ sam_global_opt_help(stderr, "--..-@-.");
return 1;
}
switch (hts_get_format(in)->format) {
case bam:
sam_close(in);
- if (bam_cat(infns_size+nargv_fns, infns, h, outfn? outfn : "-") < 0)
+ if (bam_cat(infns_size+nargv_fns, infns, h, outfn? outfn : "-", arg_list, no_pg) < 0)
ret = 1;
break;
case cram:
sam_close(in);
- if (cram_cat(infns_size+nargv_fns, infns, h, outfn? outfn : "-") < 0)
+ if (cram_cat(infns_size+nargv_fns, infns, h, outfn? outfn : "-", &ga, arg_list, no_pg) < 0)
ret = 1;
break;
free(outfn);
free(infns);
-
+ free(arg_list);
if (h)
- bam_hdr_destroy(h);
+ sam_hdr_destroy(h);
return ret;
}
/* bam_cat.c -- efficiently concatenates bam files.
- Copyright (C) 2008-2009, 2011-2013, 2015-2016 Genome Research Ltd.
+ Copyright (C) 2008-2009, 2011-2013, 2015-2017, 2019 Genome Research Ltd.
Modified SAMtools work copyright (C) 2010 Illumina, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
#include "htslib/bgzf.h"
#include "htslib/sam.h"
#include "htslib/cram.h"
-#include "htslib/khash.h"
+#include "htslib/kstring.h"
#include "samtools.h"
-
-KHASH_MAP_INIT_STR(s2i, int)
-
-// Bi-directional lookup.
-// We can go from name to ID or ID to name.
-typedef struct khash_s2i {
- khash_t(s2i) *h;
- int n_id, a_id;
- const char **id; // map Nth entry back to key
- const char **line;
-} khash_s2i;
-
-static int hash_s2i_inc(khash_s2i *hash, const char *str, const char *line, int *added) {
- // loosly based on khash_str2int_inc
- khint_t k;
- int n;
-
- if ( !hash ) return -1;
- // inefficient, but works
- char *my_str = strdup(str);
- k = kh_put(s2i, hash->h, my_str, added);
- if (*added == 0) {
- free(my_str);
- return kh_val(hash->h, k);
- }
- n = hash->n_id++;
- kh_val(hash->h, k) = n;
- if (hash->a_id <= n) {
- const char **id;
- hash->a_id = (n+1)*2;
- if (!(id = realloc(hash->id, hash->a_id*sizeof(*hash->id))))
- return -1;
- hash->id = id;
- if (!(id = realloc(hash->line, hash->a_id*sizeof(*hash->line))))
- return -1;
- hash->line = id;
- }
- hash->id[n] = my_str; // reverse map
- if (line)
- hash->line[n] = line;
-
- return n;
-}
-
-khash_s2i *hash_s2i_create(void) {
- khash_s2i *h = calloc(1, sizeof(*h));
- if (!h)
- return NULL;
-
- h->h = kh_init(s2i);
- if (!h->h) {
- free(h);
- return NULL;
- }
- return h;
-}
-
-static void hash_s2i_free(khash_s2i *hash) {
- // based on khash_str2int_destroy_free
- khint_t k;
- if (!hash) return;
- if (hash->h) {
- for (k = 0; k < kh_end(hash->h); ++k)
- if (kh_exist(hash->h, k)) free((char*)kh_key(hash->h, k));
- kh_destroy(s2i, hash->h);
- }
- if (hash->id)
- free(hash->id);
- if (hash->line)
- free(hash->line);
-
- free(hash);
-}
-
-static khash_s2i *hash_rg(const bam_hdr_t *h) {
- khash_s2i *rg2id = hash_s2i_create();
- char *cp, *line;
- int j, l;
-
- if (!h)
- return rg2id;
-
- if (!rg2id)
- return NULL;
-
- cp = h->text;
-
- for (l = 0; l+3 < h->l_text; l++) {
- line = &cp[l];
- if (!(cp[l] == '@' && cp[l+1] == 'R' && cp[l+2] == 'G')) {
- while (l < h->l_text && cp[l] != '\n')
- l++;
- continue;
- }
-
- // Found an @RG line; add to hash
- while (cp[l] != '\n') {
- while (l < h->l_text && cp[l] != '\n' && cp[l] != '\t')
- l++;
- if (l+4 < h->l_text && cp[l+1] == 'I' && cp[l+2] == 'D')
- break;
- }
- if (cp[l] == '\n')
- continue;
- l = (j = l+4);
- while (l < h->l_text && cp[l] != '\n' && cp[l] != '\t')
- l++;
-
- // To do: save id and keep realloc as needed, as hash_s2i_inc strdups.
- char *id = malloc(l-j+1);
- strncpy(id, &cp[j], l-j);
- id[l-j] = 0;
-
- int added;
- hash_s2i_inc(rg2id, id, line, &added);
- free(id);
-
- while (l < h->l_text && cp[l] != '\n')
- l++;
- }
-
- return rg2id;
-}
+#include "sam_opts.h"
/*
* Check the files are consistent and capable of being concatenated.
- * Also fills out the rg2id read-group hash and the version numbers
- * and produces a new bam_hdr_t structure with merged RG lines.
- * Note it is only a simple merge, as we lack the niceties of a proper
- * header API.
+ * Also fills out the version numbers and produces a new sam_hdr_t
+ * structure with merged RG lines.
+ * Note it is only a simple merge.
*
* Returns updated header on success;
* NULL on failure.
*/
-static bam_hdr_t *cram_cat_check_hdr(int nfn, char * const *fn, const bam_hdr_t *h,
- khash_s2i **rg2id, int *vers_maj_p, int *vers_min_p) {
+static sam_hdr_t *cram_cat_check_hdr(int nfn, char * const *fn, const sam_hdr_t *h,
+ int *vers_maj_p, int *vers_min_p) {
int i, vers_maj = -1, vers_min = -1;
- bam_hdr_t *new_h = NULL;
+ sam_hdr_t *new_h = NULL, *old_h = NULL;
+ samFile *in = NULL;
+ kstring_t ks = KS_INITIALIZE;
if (h) {
- new_h = bam_hdr_dup(h);
- *rg2id = hash_rg(new_h);
+ new_h = sam_hdr_dup(h);
+ if (!new_h) {
+ fprintf(samtools_stderr, "[%s] ERROR: header duplication failed.\n",
+ __func__);
+ goto fail;
+ }
}
for (i = 0; i < nfn; ++i) {
- samFile *in;
cram_fd *in_c;
- khint_t ki;
- int new_rg = -1;
+ int ki;
in = sam_open(fn[i], "rc");
if (in == 0) {
print_error_errno("cat", "fail to open file '%s'", fn[i]);
- return NULL;
+ goto fail;
}
in_c = in->fp.cram;
(vers_min != -1 && vers_min != vmin)) {
fprintf(samtools_stderr, "[%s] ERROR: input files have differing version numbers.\n",
__func__);
- return NULL;
+ goto fail;
}
vers_maj = vmaj;
vers_min = vmin;
- bam_hdr_t *old = sam_hdr_read(in);
- khash_s2i *rg2id_in = hash_rg(old);
+ old_h = sam_hdr_read(in);
+ if (!old_h) {
+ fprintf(samtools_stderr, "[%s] ERROR: header reading for file '%s' filed.\n",
+ __func__, fn[i]);
+ goto fail;
+ }
if (!new_h) {
- new_h = bam_hdr_dup(old);
- *rg2id = hash_rg(new_h);
+ new_h = sam_hdr_dup(old_h);
+ if (!new_h) {
+ fprintf(samtools_stderr, "[%s] ERROR: header duplication for file '%s' failed.\n",
+ __func__, fn[i]);
+ goto fail;
+ }
+ sam_hdr_destroy(old_h);
+ sam_close(in);
+ continue;
}
- // Add any existing @RG entries to our global @RG hash.
- for (ki = 0; ki < rg2id_in->n_id; ki++) {
- int added;
-
- new_rg = hash_s2i_inc(*rg2id, rg2id_in->id[ki], rg2id_in->line[ki], &added);
- //fprintf(samtools_stderr, "RG %s: #%d -> #%d\n",
- // rg2id_in->id[ki], ki, new_rg);
-
- if (added) {
- // Also add to new_h
- const char *line = rg2id_in->line[ki];
- const char *line_end = line;
- while (*line && *line_end++ != '\n')
- ;
- new_h->l_text += line_end - line;
- new_h->text = realloc(new_h->text, new_h->l_text+1);
- strncat(&new_h->text[new_h->l_text - (line_end - line)],
- line, line_end - line);
+ int old_count = sam_hdr_count_lines(old_h, "RG");
+ for (ki = 0; ki < old_count; ki++) {
+ const char *old_name = sam_hdr_line_name(old_h, "RG", ki);
+ if (old_name) {
+ int new_i = sam_hdr_line_index(new_h, "RG", old_name);
+ if (-1 == new_i) { // line does not exist in the new header
+ if (sam_hdr_find_line_pos(old_h, "RG", ki, &ks) ||
+ !ks.s || sam_hdr_add_lines(new_h, ks.s, ks.l)) {
+ fprintf(samtools_stderr, "[%s] ERROR: failed to add @RG line 'ID:%s' from file '%s'\n",
+ __func__, old_name, fn[i]);
+ goto fail;
+ }
+ ks_free(&ks);
+ }
+ } else {
+ fprintf(samtools_stderr, "[%s] ERROR: failed to read %d @RG line from file '%s'\n",
+ __func__, ki, fn[i]);
+ goto fail;
}
+ }
- if (new_rg != ki && rg2id_in->n_id > 1) {
- fprintf(samtools_stderr, "[%s] ERROR: Same size @RG lists but differing order / contents\n",
- __func__);
- return NULL;
+ if (old_count > 1 && sam_hdr_count_lines(new_h, "RG") == old_count) {
+ for (ki = 0; ki < old_count; ki++) {
+ const char *old_name = sam_hdr_line_name(old_h, "RG", ki);
+ const char *new_name = sam_hdr_line_name(new_h, "RG", ki);
+ if (!old_name || !new_name || strcmp(old_name, new_name)) {
+ fprintf(samtools_stderr, "[%s] ERROR: Same size @RG lists but differing order / contents\n",
+ __func__);
+ goto fail;
+ }
}
}
- hash_s2i_free(rg2id_in);
- bam_hdr_destroy(old);
+ sam_hdr_destroy(old_h);
sam_close(in);
}
+ ks_free(&ks);
+
*vers_maj_p = vers_maj;
*vers_min_p = vers_min;
return new_h;
+
+fail:
+ ks_free(&ks);
+ if (old_h) sam_hdr_destroy(old_h);
+ if (new_h) sam_hdr_destroy(new_h);
+ if (in) sam_close(in);
+
+ return NULL;
}
* huffman code. In this situation we can change the meta-data in the
* compression header to renumber an RG value..
*/
-int cram_cat(int nfn, char * const *fn, const bam_hdr_t *h, const char* outcram)
+int cram_cat(int nfn, char * const *fn, const sam_hdr_t *h, const char* outcram, sam_global_args *ga, char *arg_list, int no_pg)
{
samFile *out;
cram_fd *out_c;
int i, vers_maj, vers_min;
- khash_s2i *rg2id = NULL;
- bam_hdr_t *new_h = NULL;
+ sam_hdr_t *new_h = NULL;
/* Check consistent versioning and compatible headers */
- if (!(new_h = cram_cat_check_hdr(nfn, fn, h, &rg2id, &vers_maj, &vers_min)))
+ if (!(new_h = cram_cat_check_hdr(nfn, fn, h, &vers_maj, &vers_min)))
return -1;
/* Open the file with cram_vers */
char vers[100];
sprintf(vers, "%d.%d", vers_maj, vers_min);
- out = sam_open(outcram, "wc");
+ out = sam_open_format(outcram, "wc", &ga->out);
if (out == 0) {
print_error_errno("cat", "fail to open output file '%s'", outcram);
return -1;
cram_set_option(out_c, CRAM_OPT_VERSION, vers);
//fprintf(samtools_stderr, "Creating cram vers %s\n", vers);
- cram_fd_set_header(out_c, sam_hdr_parse_(new_h->text, new_h->l_text)); // needed?
+ if (!no_pg && sam_hdr_add_pg(new_h, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))
+ return -1;
+
if (sam_hdr_write(out, new_h) < 0) {
print_error_errno("cat", "Couldn't write header");
return -1;
samFile *in;
cram_fd *in_c;
cram_container *c;
- bam_hdr_t *old;
+ sam_hdr_t *old_h;
int new_rg = -1;
in = sam_open(fn[i], "rc");
}
in_c = in->fp.cram;
- old = sam_hdr_read(in);
- khash_s2i *rg2id_in = hash_rg(old);
+ old_h = sam_hdr_read(in);
+ if (!old_h) {
+ print_error("cat", "fail to read the header of file '%s'", fn[i]);
+ return -1;
+ }
// Compute RG mapping if suitable for changing.
- if (rg2id_in->n_id == 1) {
- int _;
- new_rg = hash_s2i_inc(rg2id, rg2id_in->id[0], NULL, &_);
+ if (sam_hdr_count_lines(old_h, "RG") == 1) {
+ const char *old_name = sam_hdr_line_name(old_h, "RG", 0);
+ if (old_name) {
+ new_rg = sam_hdr_line_index(new_h, "RG", old_name);
+ if (new_rg < 0) {
+ print_error("cat", "fail to find @RG line '%s' in the new header", old_name);
+ return -1;
+ }
+ } else {
+ print_error("cat", "fail to find @RG line in file '%s'", fn[i]);
+ return -1;
+ }
} else {
new_rg = 0;
}
- hash_s2i_free(rg2id_in);
-
-
// Copy contains and blocks within them
while ((c = cram_read_container(in_c))) {
cram_block *blk;
cram_free_container(c);
}
- bam_hdr_destroy(old);
+ sam_hdr_destroy(old_h);
sam_close(in);
}
sam_close(out);
-
- hash_s2i_free(rg2id);
- bam_hdr_destroy(new_h);
+ sam_hdr_destroy(new_h);
return 0;
}
#define BGZF_EMPTY_BLOCK_SIZE 28
-int bam_cat(int nfn, char * const *fn, const bam_hdr_t *h, const char* outbam)
+int bam_cat(int nfn, char * const *fn, sam_hdr_t *h, const char* outbam, char *arg_list, int no_pg)
{
BGZF *fp, *in = NULL;
uint8_t *buf = NULL;
return -1;
}
if (h) {
+ if (!no_pg && sam_hdr_add_pg(h, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))
+ goto fail;
+
if (bam_hdr_write(fp, h) < 0) {
print_error_errno("cat", "Couldn't write header");
goto fail;
goto fail;
}
for(i = 0; i < nfn; ++i){
- bam_hdr_t *old;
+ sam_hdr_t *old;
int len,j;
in = strcmp(fn[i], "-")? bgzf_open(fn[i], "r") : bgzf_fdopen(fileno(stdin), "r");
goto fail;
}
if (h == 0 && i == 0) {
+ if (!no_pg && sam_hdr_add_pg(old, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))
+ goto fail;
+
if (bam_hdr_write(fp, old) < 0) {
print_error_errno("cat", "Couldn't write header");
goto fail;
if (bgzf_raw_write(fp, ebuf, es) < 0) goto write_fail;
}
}
- bam_hdr_destroy(old);
+ sam_hdr_destroy(old);
bgzf_close(in);
in = NULL;
}
int main_cat(int argc, char *argv[])
{
- bam_hdr_t *h = 0;
+ sam_hdr_t *h = 0;
char *outfn = 0;
char **infns = NULL; // files to concatenate
int infns_size = 0;
- int c, ret = 0;
+ int c, ret = 0, no_pg = 0;
samFile *in;
+ sam_global_args ga;
+
+ static const struct option lopts[] = {
+ SAM_OPT_GLOBAL_OPTIONS('-', '-', '-', 0, '-', '@'),
+ {"no-PG", no_argument, NULL, 1},
+ { NULL, 0, NULL, 0 }
+ };
+
+ char *arg_list = NULL;
- while ((c = getopt(argc, argv, "h:o:b:")) >= 0) {
+ sam_global_args_init(&ga);
+
+ while ((c = getopt_long(argc, argv, "h:o:b:", lopts, NULL)) >= 0) {
switch (c) {
case 'h': {
samFile *fph = sam_open(optarg, "r");
}
break;
}
+ case 1:
+ no_pg = 1;
+ break;
+ default:
+ if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
}
}
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("cat", "failed to create arg_list");
+ return 1;
+ }
+
// Append files specified in argv to the list.
int nargv_fns = argc - optind;
if (nargv_fns > 0) {
fprintf(samtools_stderr, "Options: -b FILE list of input BAM/CRAM file names, one per line\n");
fprintf(samtools_stderr, " -h FILE copy the header from FILE [default is 1st input file]\n");
fprintf(samtools_stderr, " -o FILE output BAM/CRAM\n");
+ fprintf(samtools_stderr, " --no-PG do not add a PG line\n");
+ sam_global_opt_help(samtools_stderr, "--..-@-.");
return 1;
}
switch (hts_get_format(in)->format) {
case bam:
sam_close(in);
- if (bam_cat(infns_size+nargv_fns, infns, h, outfn? outfn : "-") < 0)
+ if (bam_cat(infns_size+nargv_fns, infns, h, outfn? outfn : "-", arg_list, no_pg) < 0)
ret = 1;
break;
case cram:
sam_close(in);
- if (cram_cat(infns_size+nargv_fns, infns, h, outfn? outfn : "-") < 0)
+ if (cram_cat(infns_size+nargv_fns, infns, h, outfn? outfn : "-", &ga, arg_list, no_pg) < 0)
ret = 1;
break;
free(outfn);
free(infns);
-
+ free(arg_list);
if (h)
- bam_hdr_destroy(h);
+ sam_hdr_destroy(h);
return ret;
}
--- /dev/null
+/* bam_fastq.c -- FASTA and FASTQ file generation
+
+ Copyright (C) 2009-2017, 2019 Genome Research Ltd.
+ Portions copyright (C) 2009, 2011, 2012 Broad Institute.
+
+ Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notices and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE. */
+
+#include <config.h>
+
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <stdbool.h>
+#include <ctype.h>
+#include <assert.h>
+#include <inttypes.h>
+#include <unistd.h>
+
+#include "htslib/sam.h"
+#include "htslib/klist.h"
+#include "htslib/kstring.h"
+#include "htslib/bgzf.h"
+#include "htslib/thread_pool.h"
+#include "samtools.h"
+#include "sam_opts.h"
+
+#define taglist_free(p)
+KLIST_INIT(ktaglist, char*, taglist_free)
+
+#define DEFAULT_BARCODE_TAG "BC"
+#define DEFAULT_QUALITY_TAG "QT"
+#define INDEX_SEPARATOR "+"
+
+int8_t seq_comp_table[16] = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 };
+static const char *copied_tags[] = { "RG", "BC", "QT", NULL };
+
+static void bam2fq_usage(FILE *to, const char *command)
+{
+ int fq = strcasecmp("fastq", command) == 0 || strcasecmp("bam2fq", command) == 0;
+ fprintf(to,
+"Usage: samtools %s [options...] <in.bam>\n", command);
+ fprintf(to,
+"\n"
+"Description:\n"
+"Converts a SAM, BAM or CRAM into either FASTQ or FASTA format depending on the command invoked.\n"
+"\n"
+"Options:\n"
+" -0 FILE write reads designated READ_OTHER to FILE\n"
+" -1 FILE write reads designated READ1 to FILE\n"
+" -2 FILE write reads designated READ2 to FILE\n"
+" -o FILE write reads designated READ1 or READ2 to FILE\n"
+" note: if a singleton file is specified with -s, only\n"
+" paired reads will be written to the -1 and -2 files.\n"
+" -f INT only include reads with all of the FLAGs in INT present [0]\n" // F&x == x
+" -F INT only include reads with none of the FLAGS in INT present [0x900]\n" // F&x == 0
+" -G INT only EXCLUDE reads with all of the FLAGs in INT present [0]\n" // !(F&x == x)
+" -n don't append /1 and /2 to the read name\n"
+" -N always append /1 and /2 to the read name\n");
+ if (fq) fprintf(to,
+" -O output quality in the OQ tag if present\n");
+ fprintf(to,
+" -s FILE write singleton reads designated READ1 or READ2 to FILE\n"
+" -t copy RG, BC and QT tags to the %s header line\n",
+ fq ? "FASTQ" : "FASTA");
+ fprintf(to,
+" -T TAGLIST copy arbitrary tags to the %s header line\n",
+ fq ? "FASTQ" : "FASTA");
+ if (fq) fprintf(to,
+" -v INT default quality score if not given in file [1]\n"
+" -i add Illumina Casava 1.8 format entry to header (eg 1:N:0:ATCACG)\n"
+" -c compression level [0..9] to use when creating gz or bgzf fastq files [1]\n"
+" --i1 FILE write first index reads to FILE\n"
+" --i2 FILE write second index reads to FILE\n"
+" --barcode-tag TAG Barcode tag [default: " DEFAULT_BARCODE_TAG "]\n"
+" --quality-tag TAG Quality tag [default: " DEFAULT_QUALITY_TAG "]\n"
+" --index-format STR How to parse barcode and quality tags\n\n");
+ sam_global_opt_help(to, "-.--.@-.");
+ fprintf(to,
+"\n"
+"The files will be automatically compressed if the file names have a .gz or .bgzf extension.\n"
+"The input to this program must be collated by name. Run 'samtools collate' or 'samtools sort -n'.\n"
+"\n"
+"Reads are designated READ1 if FLAG READ1 is set and READ2 is not set.\n"
+"Reads are designated READ2 if FLAG READ1 is not set and READ2 is set.\n"
+"Reads are designated READ_OTHER if FLAGs READ1 and READ2 are either both set\n"
+"or both unset.\n"
+"Run 'samtools flags' for more information on flag codes and meanings.\n");
+ fprintf(to,
+"\n"
+"The index-format string describes how to parse the barcode and quality tags, for example:\n"
+" i14i8 the first 14 characters are index 1, the next 8 characters are index 2\n"
+" n8i14 ignore the first 8 characters, and use the next 14 characters for index 1\n"
+"If the tag contains a separator, then the numeric part can be replaced with '*' to mean\n"
+"'read until the separator or end of tag', for example:\n"
+" n*i* ignore the left part of the tag until the separator, then use the second part\n"
+" of the tag as index 1\n");
+ fprintf(to,
+"\n"
+"Examples:\n"
+" To get just the paired reads in separate files, use:\n"
+" samtools %s -1 paired1.%s -2 paired2.%s -0 /dev/null -s /dev/null -n in.bam\n"
+"\n To get all non-supplementary/secondary reads in a single file, redirect the output:\n"
+" samtools %s in.bam > all_reads.%s\n",
+ command, fq ? "fq" : "fa", fq ? "fq" : "fa",
+ command, fq ? "fq" : "fa");
+}
+
+typedef enum { READ_UNKNOWN = 0, READ_1 = 1, READ_2 = 2 } readpart;
+typedef enum { FASTA, FASTQ } fastfile;
+typedef struct bam2fq_opts {
+ char *fnse;
+ char *fnr[3];
+ char *fn_input; // pointer to input filename in argv do not free
+ bool has12, has12always, use_oq, copy_tags, illumina_tag;
+ int flag_on, flag_off, flag_alloff;
+ sam_global_args ga;
+ fastfile filetype;
+ int def_qual;
+ char *barcode_tag;
+ char *quality_tag;
+ char *index_file[2];
+ char *index_format;
+ char *extra_tags;
+ char compression_level;
+} bam2fq_opts_t;
+
+typedef struct bam2fq_state {
+ samFile *fp;
+ BGZF *fpse;
+ BGZF *fpr[3];
+ BGZF *fpi[2];
+ BGZF *hstdout;
+ sam_hdr_t *h;
+ bool has12, use_oq, copy_tags, illumina_tag;
+ int flag_on, flag_off, flag_alloff;
+ fastfile filetype;
+ int def_qual;
+ klist_t(ktaglist) *taglist;
+ char *index_sequence;
+ char compression_level;
+ htsThreadPool p;
+} bam2fq_state_t;
+
+/*
+ * Get and decode the read from a BAM record.
+ *
+ * TODO: htslib really needs an interface for this. Consider this or perhaps
+ * bam_get_seq_str (current vs original orientation) and bam_get_qual_str
+ * functions as string formatted equivalents to bam_get_{seq,qual}?
+ */
+
+/*
+ * Reverse a string in place.
+ * From http://stackoverflow.com/questions/8534274/is-the-strrev-function-not-available-in-linux.
+ * Author Sumit-naik: http://stackoverflow.com/users/4590926/sumit-naik
+ */
+static char *reverse(char *str)
+{
+ int i = strlen(str)-1,j=0;
+ char ch;
+ while (i>j) {
+ ch = str[i];
+ str[i]= str[j];
+ str[j] = ch;
+ i--;
+ j++;
+ }
+ return str;
+}
+
+/* return the read, reverse complemented if necessary */
+static char *get_read(const bam1_t *rec)
+{
+ int len = rec->core.l_qseq + 1;
+ char *read = calloc(1, len);
+ char *seq = (char *)bam_get_seq(rec);
+ int n;
+
+ if (!read) return NULL;
+
+ for (n=0; n < rec->core.l_qseq; n++) {
+ if (rec->core.flag & BAM_FREVERSE) read[n] = seq_nt16_str[seq_comp_table[bam_seqi(seq,n)]];
+ else read[n] = seq_nt16_str[bam_seqi(seq,n)];
+ }
+ if (rec->core.flag & BAM_FREVERSE) reverse(read);
+ return read;
+}
+
+/*
+ * get and decode the quality from a BAM record
+ */
+static int get_quality(const bam1_t *rec, char **qual_out)
+{
+ char *quality = calloc(1, rec->core.l_qseq + 1);
+ char *q = (char *)bam_get_qual(rec);
+ int n;
+
+ if (!quality) return -1;
+
+ if (*q == '\xff') {
+ free(quality);
+ *qual_out = NULL;
+ return 0;
+ }
+
+ for (n=0; n < rec->core.l_qseq; n++) {
+ quality[n] = q[n]+33;
+ }
+ if (rec->core.flag & BAM_FREVERSE) reverse(quality);
+ *qual_out = quality;
+ return 0;
+}
+
+//
+// End of htslib complaints
+//
+
+
+static readpart which_readpart(const bam1_t *b)
+{
+ if ((b->core.flag & BAM_FREAD1) && !(b->core.flag & BAM_FREAD2)) {
+ return READ_1;
+ } else if ((b->core.flag & BAM_FREAD2) && !(b->core.flag & BAM_FREAD1)) {
+ return READ_2;
+ } else {
+ return READ_UNKNOWN;
+ }
+}
+
+/*
+ * parse the length part from the index-format string
+ */
+static int getLength(char **s)
+{
+ int n = 0;
+ while (**s) {
+ if (**s == '*') { n=-1; (*s)++; break; }
+ if ( !isdigit(**s)) break;
+ n = n*10 + ((**s)-'0');
+ (*s)++;
+ }
+ return n;
+}
+
+static bool copy_tag(const char *tag, const bam1_t *rec, kstring_t *linebuf)
+{
+ uint8_t *s = bam_aux_get(rec, tag);
+ if (s) {
+ char aux_type = *s;
+ switch (aux_type) {
+ case 'C':
+ case 'S': aux_type = 'I'; break;
+ case 'c':
+ case 's': aux_type = 'i'; break;
+ case 'd': aux_type = 'f'; break;
+ }
+
+ // Ensure space. Need 6 chars + length of tag. Max length of
+ // i is 16, A is 21, B currently 26, Z is unknown, so
+ // have to check that one later.
+ if (ks_resize(linebuf, ks_len(linebuf) + 64) < 0) return false;
+
+ kputc('\t', linebuf);
+ kputsn(tag, 2, linebuf);
+ kputc(':', linebuf);
+ kputc(aux_type=='I'? 'i': aux_type, linebuf);
+ kputc(':', linebuf);
+ switch (aux_type) {
+ case 'H':
+ case 'Z':
+ if (kputs(bam_aux2Z(s), linebuf) < 0) return false;
+ break;
+ case 'i': kputw(bam_aux2i(s), linebuf); break;
+ case 'I': kputuw(bam_aux2i(s), linebuf); break;
+ case 'A': kputc(bam_aux2A(s), linebuf); break;
+ case 'f': kputd(bam_aux2f(s), linebuf); break;
+ case 'B': kputs("*** Unhandled aux type ***", linebuf); return false;
+ default: kputs("*** Unknown aux type ***", linebuf); return false;
+ }
+ }
+ return true;
+}
+
+static int insert_index_sequence_into_linebuf(char *index_sequence, kstring_t *linebuf, bam1_t *rec)
+{
+ if (!index_sequence) return 0;
+
+ kstring_t new = {0,0,NULL};
+ if (linebuf->s) {
+ char *s = strchr(linebuf->s, '\n');
+ if (s) {
+ if (ks_resize(&new, linebuf->l + strlen(index_sequence) + 16) < 0)
+ return -1;
+ *s = 0;
+ kputs(linebuf->s, &new);
+ kputc(' ', &new);
+ readpart readpart = which_readpart(rec);
+ if (readpart == READ_1) kputc('1', &new);
+ else if (readpart == READ_2) kputc('2', &new);
+ else kputc('0', &new);
+
+ kputc(':', &new);
+ if (rec->core.flag & BAM_FQCFAIL) kputc('Y', &new);
+ else kputc('N', &new);
+
+ kputs(":0:", &new);
+ kputs(index_sequence, &new);
+ kputc('\n', &new);
+ kputs(s+1, &new);
+ free(ks_release(linebuf));
+ linebuf->s = new.s; linebuf->l = new.l; linebuf->m = new.m;
+ }
+ }
+ return 0;
+}
+
+static bool make_fq_line(const bam1_t *rec, char *seq, char *qual, kstring_t *linebuf, const bam2fq_state_t *state)
+{
+ int i;
+
+ linebuf->l = 0;
+ // Write read name
+ if (kputc(state->filetype == FASTA? '>' : '@', linebuf) < 0) return false;
+ if (kputs(bam_get_qname(rec), linebuf) < 0) return false;
+ // Add the /1 /2 if requested
+ if (state->has12) {
+ readpart readpart = which_readpart(rec);
+ if (readpart == READ_1) {
+ if (kputs("/1", linebuf) < 0) return false;
+ } else if (readpart == READ_2) {
+ if (kputs("/2", linebuf) < 0) return false;
+ }
+ }
+ if (state->copy_tags) {
+ for (i = 0; copied_tags[i]; ++i) {
+ if (!copy_tag(copied_tags[i], rec, linebuf)) {
+ fprintf(stderr, "Problem copying aux tags: [%s]\n", linebuf->s);
+ return false;
+ }
+ }
+ }
+
+ if (state->taglist->size) {
+ kliter_t(ktaglist) *p;
+ for (p = kl_begin(state->taglist); p != kl_end(state->taglist); p = kl_next(p)) {
+ if (!copy_tag(kl_val(p), rec, linebuf)) {
+ fprintf(stderr, "Problem copying aux tags: [%s]\n", linebuf->s);
+ return false;
+ }
+ }
+ }
+
+ if (kputc('\n', linebuf) < 0) return false;
+ if (kputs(seq, linebuf) < 0) return false;
+ if (kputc('\n', linebuf) < 0) return false;
+
+ if (state->filetype == FASTQ) {
+ // Write quality
+ if (kputs("+\n", linebuf) < 0) return false;
+ if (qual && *qual) {
+ if (kputs(qual, linebuf) < 0) return false;
+ } else {
+ int len = strlen(seq);
+ if (ks_resize(linebuf, ks_len(linebuf) + len + 1) < 0) return false;
+ for (i = 0; i < len; ++i) {
+ kputc(33 + state->def_qual, linebuf);
+ }
+ }
+ if (kputc('\n', linebuf) < 0) return false;
+ }
+ return true;
+}
+
+/*
+ * Create FASTQ lines from the barcode tag using the index-format
+ */
+static bool tags2fq(bam1_t *rec, bam2fq_state_t *state, const bam2fq_opts_t* opts)
+{
+ uint8_t *p;
+ char *ifmt = opts->index_format;
+ char *tag = NULL;
+ char *qual = NULL;
+ char *sub_tag = NULL;
+ char *sub_qual = NULL;
+ size_t tag_len;
+ int file_number = 0;
+ kstring_t linebuf = { 0, 0, NULL }; // Buffer
+
+ if (!ifmt) return true;
+
+ // read barcode tag
+ p = bam_aux_get(rec,opts->barcode_tag);
+ if (p) tag = bam_aux2Z(p);
+
+ if (!tag) return true; // there is no tag
+
+ tag_len = strlen(tag);
+ sub_tag = calloc(1, tag_len + 1);
+ if (!sub_tag) goto fail;
+ sub_qual = calloc(1, tag_len + 1);
+ if (!sub_qual) goto fail;
+
+ // read quality tag
+ p = bam_aux_get(rec, opts->quality_tag);
+ if (p) qual = bam_aux2Z(p);
+
+ // Parse the index-format string
+ while (*ifmt) {
+ if (file_number > 1) break; // shouldn't happen if we've validated paramaters correctly
+ char action = *ifmt; // should be 'i' or 'n'
+ ifmt++; // skip over action
+ int index_len = getLength(&ifmt);
+ int n = 0;
+
+ if (index_len < 0) {
+ // read until separator
+ while (isalpha(*tag)) {
+ sub_tag[n] = *tag++;
+ if (qual) sub_qual[n] = *qual++;
+ n++;
+ }
+ if (*tag) { // skip separator
+ tag++;
+ if (qual) qual++;
+ }
+ } else {
+ // read index_len characters
+ while (index_len-- && *tag) {
+ sub_tag[n] = *tag++;
+ if (qual) sub_qual[n] = *qual++;
+ n++;
+ }
+ }
+ sub_tag[n] = '\0';
+ sub_qual[n] = '\0';
+
+ if (action=='i' && *sub_tag) {
+ if (state->index_sequence) {
+ char *new_index_sequence = realloc(state->index_sequence, strlen(state->index_sequence) + strlen(sub_tag) + 2);
+ if (!new_index_sequence) goto fail;
+ state->index_sequence = new_index_sequence;
+ strcat(state->index_sequence, INDEX_SEPARATOR);
+ strcat(state->index_sequence, sub_tag);
+ } else {
+ state->index_sequence = strdup(sub_tag); // we're going to need this later...
+ }
+ if (!state->index_sequence) goto fail;
+ if (!make_fq_line(rec, sub_tag, sub_qual, &linebuf, state)) goto fail;
+ if (state->illumina_tag) {
+ if (insert_index_sequence_into_linebuf(sub_tag, &linebuf, rec) < 0) {
+ goto fail;
+ }
+ }
+ if (state->fpi[file_number]) {
+ if (bgzf_write(state->fpi[file_number++], linebuf.s, linebuf.l) < 0)
+ goto fail;
+ }
+ }
+
+ }
+
+ free(sub_qual); free(sub_tag);
+ free(linebuf.s);
+ return true;
+
+ fail:
+ perror(__func__);
+ free(sub_qual); free(sub_tag);
+ free(linebuf.s);
+ return false;
+}
+
+// Transform a bam1_t record into a string with the FASTQ representation of it
+// @returns false for error, true for success
+static bool bam1_to_fq(const bam1_t *b, kstring_t *linebuf, const bam2fq_state_t *state)
+{
+ int32_t qlen = b->core.l_qseq;
+ assert(qlen >= 0);
+ const uint8_t *oq = NULL;
+ char *qual = NULL;
+
+ char *seq = get_read(b);
+ if (!seq) return false;
+
+ if (state->use_oq) oq = bam_aux_get(b, "OQ");
+ if (oq && *oq=='Z') {
+ qual = strdup(bam_aux2Z(oq));
+ if (!qual) goto fail;
+ if (b->core.flag & BAM_FREVERSE) { // read is reverse complemented
+ reverse(qual);
+ }
+ } else {
+ if (get_quality(b, &qual) < 0) goto fail;
+ }
+
+ if (!make_fq_line(b, seq, qual, linebuf, state)) goto fail;
+
+ free(qual);
+ free(seq);
+ return true;
+
+ fail:
+ free(seq);
+ free(qual);
+ return false;
+}
+
+static void free_opts(bam2fq_opts_t *opts)
+{
+ free(opts->barcode_tag);
+ free(opts->quality_tag);
+ free(opts->index_format);
+ free(opts->extra_tags);
+ free(opts);
+}
+
+// return true if valid
+static bool parse_opts(int argc, char *argv[], bam2fq_opts_t** opts_out)
+{
+ // Parse args
+ bam2fq_opts_t* opts = calloc(1, sizeof(bam2fq_opts_t));
+ opts->has12 = true;
+ opts->has12always = false;
+ opts->filetype = FASTQ;
+ opts->def_qual = 1;
+ opts->barcode_tag = NULL;
+ opts->quality_tag = NULL;
+ opts->index_format = NULL;
+ opts->index_file[0] = NULL;
+ opts->index_file[1] = NULL;
+ opts->extra_tags = NULL;
+ opts->compression_level = 1;
+ opts->flag_off = BAM_FSECONDARY|BAM_FSUPPLEMENTARY;
+ int flag_off_set = 0;
+
+ int c;
+ sam_global_args_init(&opts->ga);
+ static const struct option lopts[] = {
+ SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0, '@'),
+ {"i1", required_argument, NULL, 1},
+ {"I1", required_argument, NULL, 1},
+ {"i2", required_argument, NULL, 2},
+ {"I2", required_argument, NULL, 2},
+ {"if", required_argument, NULL, 3},
+ {"IF", required_argument, NULL, 3},
+ {"index-format", required_argument, NULL, 3},
+ {"barcode-tag", required_argument, NULL, 'b'},
+ {"quality-tag", required_argument, NULL, 'q'},
+ { NULL, 0, NULL, 0 }
+ };
+ while ((c = getopt_long(argc, argv, "0:1:2:o:f:F:G:niNOs:c:tT:v:@:", lopts, NULL)) > 0) {
+ switch (c) {
+ case 'b': opts->barcode_tag = strdup(optarg); break;
+ case 'q': opts->quality_tag = strdup(optarg); break;
+ case 1 : opts->index_file[0] = optarg; break;
+ case 2 : opts->index_file[1] = optarg; break;
+ case 3 : opts->index_format = strdup(optarg); break;
+ case '0': opts->fnr[0] = optarg; break;
+ case '1': opts->fnr[1] = optarg; break;
+ case '2': opts->fnr[2] = optarg; break;
+ case 'o': opts->fnr[1] = optarg; opts->fnr[2] = optarg; break;
+ case 'f': opts->flag_on |= strtol(optarg, 0, 0); break;
+ case 'F':
+ if (!flag_off_set) {
+ flag_off_set = 1;
+ opts->flag_off = 0;
+ }
+ opts->flag_off |= strtol(optarg, 0, 0); break;
+ case 'G': opts->flag_alloff |= strtol(optarg, 0, 0); break;
+ case 'n': opts->has12 = false; break;
+ case 'N': opts->has12always = true; break;
+ case 'O': opts->use_oq = true; break;
+ case 's': opts->fnse = optarg; break;
+ case 't': opts->copy_tags = true; break;
+ case 'i': opts->illumina_tag = true; break;
+ case 'c': opts->compression_level = atoi(optarg); break;
+ case 'T': opts->extra_tags = strdup(optarg); break;
+ case 'v': opts->def_qual = atoi(optarg); break;
+ case '?': bam2fq_usage(stderr, argv[0]); free_opts(opts); return false;
+ default:
+ if (parse_sam_global_opt(c, optarg, lopts, &opts->ga) != 0) {
+ bam2fq_usage(stderr, argv[0]); free_opts(opts); return false;
+ }
+ break;
+ }
+ }
+
+ if (opts->fnr[1] || opts->fnr[2]) opts->has12 = false;
+ if (opts->has12always) opts->has12 = true;
+
+ if (!opts->barcode_tag) opts->barcode_tag = strdup(DEFAULT_BARCODE_TAG);
+ if (!opts->quality_tag) opts->quality_tag = strdup(DEFAULT_QUALITY_TAG);
+
+ int nIndex = 0;
+ if (opts->index_format) {
+ char *s;
+ for (s = opts->index_format; *s; s++) {
+ if (*s == 'i') nIndex++;
+ }
+ }
+ if (nIndex>2) {
+ fprintf(stderr,"Invalid index format: more than 2 indexes\n");
+ bam2fq_usage(stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ if (opts->index_file[1] && !opts->index_file[0]) {
+ fprintf(stderr, "Index one specified, but index two not given\n");
+ bam2fq_usage(stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ if (opts->illumina_tag && !nIndex) {
+ fprintf(stderr, "You must specify an index format (--index-format) with the Illumina Casava (-i) option\n");
+ bam2fq_usage(stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ if (nIndex==0 && opts->index_file[0]) {
+ fprintf(stderr, "index_format not specified, but index file given\n");
+ bam2fq_usage(stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ if (opts->def_qual < 0 || 93 < opts->def_qual) {
+ fprintf(stderr, "Invalid -v default quality %i, allowed range 0 to 93\n", opts->def_qual);
+ bam2fq_usage(stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ const char* type_str = argv[0];
+ if (strcasecmp("fastq", type_str) == 0 || strcasecmp("bam2fq", type_str) == 0) {
+ opts->filetype = FASTQ;
+ } else if (strcasecmp("fasta", type_str) == 0) {
+ opts->filetype = FASTA;
+ } else {
+ print_error("bam2fq", "Unrecognised type call \"%s\", this should be impossible... but you managed it!", type_str);
+ bam2fq_usage(stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ if (argc == optind && isatty(STDIN_FILENO)) {
+ bam2fq_usage(stdout, argv[0]);
+ free_opts(opts);
+ return true;
+ }
+
+ if (argc - optind > 1) {
+ fprintf(stderr, "Too many arguments.\n");
+ bam2fq_usage(stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+ opts->fn_input = argc > optind ? argv[optind] : "-";
+ *opts_out = opts;
+ return true;
+}
+
+static BGZF *open_fqfile(char *filename, int c, htsThreadPool *tp)
+{
+ char mode[4] = "w";
+ size_t len = strlen(filename);
+
+ mode[2] = 0; mode[3] = 0;
+ if (len > 3 && strstr(filename + (len - 3),".gz")) {
+ mode[1] = 'g'; mode[2] = c+'0';
+ } else if ((len > 4 && strstr(filename + (len - 4),".bgz"))
+ || (len > 5 && strstr(filename + (len - 5),".bgzf"))) {
+ mode[1] = c+'0';
+ } else {
+ mode[1] = 'u';
+ }
+
+ BGZF *fp = bgzf_open(filename,mode);
+ if (!fp)
+ return fp;
+ if (tp->pool && bgzf_thread_pool(fp, tp->pool, tp->qsize) < 0) {
+ bgzf_close(fp);
+ return NULL;
+ }
+ return fp;
+}
+
+static bool init_state(const bam2fq_opts_t* opts, bam2fq_state_t** state_out)
+{
+ bam2fq_state_t* state = calloc(1, sizeof(bam2fq_state_t));
+ state->flag_on = opts->flag_on;
+ state->flag_off = opts->flag_off;
+ state->flag_alloff = opts->flag_alloff;
+ state->has12 = opts->has12;
+ state->use_oq = opts->use_oq;
+ state->illumina_tag = opts->illumina_tag;
+ state->copy_tags = opts->copy_tags;
+ state->filetype = opts->filetype;
+ state->def_qual = opts->def_qual;
+ state->index_sequence = NULL;
+ state->hstdout = NULL;
+ state->compression_level = opts->compression_level;
+
+ state->taglist = kl_init(ktaglist);
+ if (opts->extra_tags) {
+ char *save_p;
+ char *s = strtok_r(opts->extra_tags, ",", &save_p);
+ while (s) {
+ if (strlen(s) != 2) {
+ fprintf(stderr, "Parsing extra tags - '%s' is not two characters\n", s);
+ free(state);
+ return false;
+ }
+ char **et = kl_pushp(ktaglist, state->taglist);
+ *et = s;
+ s = strtok_r(NULL, ",", &save_p);
+ }
+ }
+
+ state->fp = sam_open(opts->fn_input, "r");
+ if (state->fp == NULL) {
+ print_error_errno("bam2fq","Cannot read file \"%s\"", opts->fn_input);
+ free(state);
+ return false;
+ }
+
+ state->p.pool = NULL;
+ if (opts->ga.nthreads > 0) {
+ if (!(state->p.pool = hts_tpool_init(opts->ga.nthreads))) {
+ fprintf(stderr, "Failed to create thread pool\n");
+ free(state);
+ return false;
+ }
+ state->p.qsize = opts->ga.nthreads*2;
+ hts_set_thread_pool(state->fp, &state->p);
+ }
+
+ uint32_t rf = SAM_QNAME | SAM_FLAG | SAM_SEQ | SAM_QUAL;
+ if (opts->use_oq || opts->extra_tags || opts->index_file[0]) rf |= SAM_AUX;
+ if (hts_set_opt(state->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
+ fprintf(stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
+ free(state);
+ return false;
+ }
+ if (hts_set_opt(state->fp, CRAM_OPT_DECODE_MD, 0)) {
+ fprintf(stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
+ free(state);
+ return false;
+ }
+ if (opts->fnse) {
+ state->fpse = open_fqfile(opts->fnse, state->compression_level, &state->p);
+ if (state->fpse == NULL) {
+ print_error_errno("bam2fq", "Cannot write to singleton file \"%s\"", opts->fnse);
+ free(state);
+ return false;
+ }
+ }
+
+ if (opts->ga.reference) {
+ if (hts_set_fai_filename(state->fp, opts->ga.reference) != 0) {
+ print_error_errno("bam2fq", "cannot load reference \"%s\"", opts->ga.reference);
+ free(state);
+ return false;
+ }
+ }
+
+ int i, j;
+ for (i = 0; i < 3; ++i) {
+ if (opts->fnr[i]) {
+ for (j = 0; j < i; j++)
+ if (opts->fnr[j] && strcmp(opts->fnr[j], opts->fnr[i]) == 0)
+ break;
+ if (j == i) {
+ state->fpr[i] = open_fqfile(opts->fnr[i], state->compression_level, &state->p);
+ if (state->fpr[i] == NULL) {
+ print_error_errno("bam2fq", "Cannot write to r%d file \"%s\"",
+ i, opts->fnr[i]);
+ free(state);
+ return false;
+ }
+ } else {
+ state->fpr[i] = state->fpr[j];
+ }
+ } else {
+ if (!state->hstdout) {
+ state->hstdout = bgzf_dopen(fileno(stdout), "wu");
+ if (!state->hstdout) {
+ print_error_errno("bam2fq", "Cannot open STDOUT");
+ free(state);
+ return false;
+ }
+ }
+ state->fpr[i] = state->hstdout;
+ }
+ }
+ for (i = 0; i < 2; i++) {
+ state->fpi[i] = NULL;
+ if (opts->index_file[i]) {
+ for (j = 0; j < 3; j++)
+ if (opts->fnr[j] && strcmp(opts->fnr[j], opts->index_file[i]) == 0)
+ break;
+ for (j -= 3; j >= 0 && j < i; j++)
+ if (opts->index_file[j] && strcmp(opts->index_file[j], opts->index_file[i]) == 0)
+ break;
+ if (i == j) {
+ state->fpi[i] = open_fqfile(opts->index_file[i], state->compression_level, &state->p);
+ if (state->fpi[i] == NULL) {
+ print_error_errno("bam2fq", "Cannot write to i%d file \"%s\"",
+ i+1, opts->index_file[i]);
+ free(state);
+ return false;
+ }
+ } else if (j < 0) {
+ state->fpi[i] = state->fpr[j+3];
+ } else {
+ state->fpi[i] = state->fpi[j];
+ }
+ }
+ }
+
+ state->h = sam_hdr_read(state->fp);
+ if (state->h == NULL) {
+ fprintf(stderr, "Failed to read header for \"%s\"\n", opts->fn_input);
+ free(state);
+ return false;
+ }
+
+ *state_out = state;
+ return true;
+}
+
+static bool destroy_state(const bam2fq_opts_t *opts, bam2fq_state_t *state, int* status)
+{
+ bool valid = true;
+ sam_hdr_destroy(state->h);
+ check_sam_close("bam2fq", state->fp, opts->fn_input, "file", status);
+ if (state->fpse && bgzf_close(state->fpse)) { print_error_errno("bam2fq", "Error closing singleton file \"%s\"", opts->fnse); valid = false; }
+ int i, j;
+ for (i = 0; i < 3; ++i) {
+ if (state->fpr[i] != state->hstdout) {
+ for (j = 0; j < i; j++)
+ if (state->fpr[i] == state->fpr[j])
+ break;
+ if (j == i && bgzf_close(state->fpr[i])) {
+ print_error_errno("bam2fq", "Error closing r%d file \"%s\"", i, opts->fnr[i]);
+ valid = false;
+ }
+ }
+ }
+ if (state->hstdout) {
+ if (bgzf_close(state->hstdout)) {
+ print_error_errno("bam2fq", "Error closing STDOUT");
+ valid = false;
+ }
+ }
+ for (i = 0; i < 2; i++) {
+ for (j = 0; j < 3; j++)
+ if (state->fpi[i] == state->fpr[j])
+ break;
+ for (j -= 3; j >= 0 && j < i; j++)
+ if (state->fpi[i] == state->fpi[j])
+ break;
+ if (j == i && state->fpi[i] && bgzf_close(state->fpi[i])) {
+ print_error_errno("bam2fq", "Error closing i%d file \"%s\"", i+1, opts->index_file[i]);
+ valid = false;
+ }
+ }
+ kl_destroy(ktaglist,state->taglist);
+ free(state->index_sequence);
+ if (state->p.pool)
+ hts_tpool_destroy(state->p.pool);
+ free(state);
+ return valid;
+}
+
+static inline bool filter_it_out(const bam1_t *b, const bam2fq_state_t *state)
+{
+ return ((b->core.flag&(state->flag_on)) != state->flag_on // or reads indicated by filter flags
+ || (b->core.flag&(state->flag_off)) != 0
+ || (b->core.flag&(state->flag_alloff) && (b->core.flag&(state->flag_alloff)) == state->flag_alloff));
+
+}
+
+static bool bam2fq_mainloop(bam2fq_state_t *state, bam2fq_opts_t* opts)
+{
+ int n;
+ bam1_t *records[3] = {NULL, NULL, NULL};
+ char *current_qname = NULL;
+ int64_t n_reads = 0, n_singletons = 0; // Statistics
+ kstring_t linebuf[3] = {{0,0,NULL},{0,0,NULL},{0,0,NULL}};
+ int score[3];
+ int at_eof;
+ bool valid = true;
+ bam1_t* b = NULL;
+
+ while (true) {
+ if (!b)
+ b = bam_init1();
+ if (b == NULL) {
+ perror("[bam2fq_mainloop] Malloc error for bam record buffer.");
+ valid = false;
+ break;
+ }
+ int res = sam_read1(state->fp, state->h, b);
+ if (res < -1) {
+ fprintf(stderr, "[bam2fq_mainloop] Failed to read bam record.\n");
+ valid = false;
+ break;
+ }
+ at_eof = res < 0;
+
+ if (!at_eof && filter_it_out(b, state))
+ continue;
+ if (!at_eof) ++n_reads;
+
+ if (at_eof || !current_qname || (strcmp(current_qname, bam_get_qname(b)) != 0)) {
+ if (current_qname) {
+ if (state->illumina_tag) {
+ for (n=0; valid && n<3; n++) {
+ if (!records[n]) continue;
+ if (insert_index_sequence_into_linebuf(state->index_sequence, &linebuf[n], records[n]) < 0) valid = false;
+ }
+ if (!valid) break;
+ }
+ free(state->index_sequence); state->index_sequence = NULL;
+ if (score[1] > 0 && score[2] > 0) {
+ // print linebuf[1] to fpr[1], linebuf[2] to fpr[2]
+ if (bgzf_write(state->fpr[1], linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
+ if (bgzf_write(state->fpr[2], linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
+ } else if (score[1] > 0 || score[2] > 0) {
+ if (state->fpse) {
+ // print whichever one exists to fpse
+ if (score[1] > 0) {
+ if (bgzf_write(state->fpse, linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
+ } else {
+ if (bgzf_write(state->fpse, linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
+ }
+ ++n_singletons;
+ } else {
+ if (score[1] > 0) {
+ if (bgzf_write(state->fpr[1], linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
+ } else {
+ if (bgzf_write(state->fpr[2], linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
+ }
+ }
+ }
+ if (score[0]) { // TODO: check this
+ // print linebuf[0] to fpr[0]
+ if (bgzf_write(state->fpr[0], linebuf[0].s, linebuf[0].l) < 0) { valid = false; break; }
+ }
+ }
+
+
+ free(current_qname); current_qname = NULL;
+ score[0] = score[1] = score[2] = 0;
+ for (n=0; n < 3; n++) {
+ bam_destroy1(records[n]); records[n]=NULL;
+ }
+
+ if (at_eof) { break; }
+
+ current_qname = strdup(bam_get_qname(b));
+ if (!current_qname) { valid = false; break; }
+ }
+
+ // Prefer a copy of the read that has base qualities
+ int b_score = bam_get_qual(b)[0] != 0xff? 2 : 1;
+ readpart rp = which_readpart(b);
+ if (b_score > score[rp]) {
+ if (!tags2fq(b, state, opts)) { valid = false; break; }
+ if (records[rp]) bam_destroy1(records[rp]);
+ records[rp] = b;
+ score[rp] = b_score;
+ b = NULL;
+ if(!bam1_to_fq(records[rp], &linebuf[rp], state)) {
+ fprintf(stderr, "[%s] Error converting read to FASTA/Q\n", __func__);
+ valid = false; break;
+ }
+ }
+ }
+ if (!valid)
+ {
+ perror("[bam2fq_mainloop] Error writing to FASTx files.");
+ }
+ bam_destroy1(b);
+ for (n=0; n < 3; n++) {
+ bam_destroy1(records[n]);
+ }
+ free(current_qname);
+ free(linebuf[0].s);
+ free(linebuf[1].s);
+ free(linebuf[2].s);
+ fprintf(stderr, "[M::%s] discarded %" PRId64 " singletons\n", __func__, n_singletons);
+ fprintf(stderr, "[M::%s] processed %" PRId64 " reads\n", __func__, n_reads);
+
+ return valid;
+}
+
+int main_bam2fq(int argc, char *argv[])
+{
+ int status = EXIT_SUCCESS;
+ bam2fq_opts_t* opts = NULL;
+ bam2fq_state_t* state = NULL;
+
+ bool valid = parse_opts(argc, argv, &opts);
+ if (!valid || opts == NULL) return valid ? EXIT_SUCCESS : EXIT_FAILURE;
+
+ if (!init_state(opts, &state)) return EXIT_FAILURE;
+
+ if (!bam2fq_mainloop(state,opts)) status = EXIT_FAILURE;
+
+ if (!destroy_state(opts, state, &status)) return EXIT_FAILURE;
+ sam_global_args_free(&opts->ga);
+ free_opts(opts);
+
+ return status;
+}
--- /dev/null
+#include "samtools.pysam.h"
+
+/* bam_fastq.c -- FASTA and FASTQ file generation
+
+ Copyright (C) 2009-2017, 2019 Genome Research Ltd.
+ Portions copyright (C) 2009, 2011, 2012 Broad Institute.
+
+ Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notices and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE. */
+
+#include <config.h>
+
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <stdbool.h>
+#include <ctype.h>
+#include <assert.h>
+#include <inttypes.h>
+#include <unistd.h>
+
+#include "htslib/sam.h"
+#include "htslib/klist.h"
+#include "htslib/kstring.h"
+#include "htslib/bgzf.h"
+#include "htslib/thread_pool.h"
+#include "samtools.h"
+#include "sam_opts.h"
+
+#define taglist_free(p)
+KLIST_INIT(ktaglist, char*, taglist_free)
+
+#define DEFAULT_BARCODE_TAG "BC"
+#define DEFAULT_QUALITY_TAG "QT"
+#define INDEX_SEPARATOR "+"
+
+int8_t seq_comp_table[16] = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 };
+static const char *copied_tags[] = { "RG", "BC", "QT", NULL };
+
+static void bam2fq_usage(FILE *to, const char *command)
+{
+ int fq = strcasecmp("fastq", command) == 0 || strcasecmp("bam2fq", command) == 0;
+ fprintf(to,
+"Usage: samtools %s [options...] <in.bam>\n", command);
+ fprintf(to,
+"\n"
+"Description:\n"
+"Converts a SAM, BAM or CRAM into either FASTQ or FASTA format depending on the command invoked.\n"
+"\n"
+"Options:\n"
+" -0 FILE write reads designated READ_OTHER to FILE\n"
+" -1 FILE write reads designated READ1 to FILE\n"
+" -2 FILE write reads designated READ2 to FILE\n"
+" -o FILE write reads designated READ1 or READ2 to FILE\n"
+" note: if a singleton file is specified with -s, only\n"
+" paired reads will be written to the -1 and -2 files.\n"
+" -f INT only include reads with all of the FLAGs in INT present [0]\n" // F&x == x
+" -F INT only include reads with none of the FLAGS in INT present [0x900]\n" // F&x == 0
+" -G INT only EXCLUDE reads with all of the FLAGs in INT present [0]\n" // !(F&x == x)
+" -n don't append /1 and /2 to the read name\n"
+" -N always append /1 and /2 to the read name\n");
+ if (fq) fprintf(to,
+" -O output quality in the OQ tag if present\n");
+ fprintf(to,
+" -s FILE write singleton reads designated READ1 or READ2 to FILE\n"
+" -t copy RG, BC and QT tags to the %s header line\n",
+ fq ? "FASTQ" : "FASTA");
+ fprintf(to,
+" -T TAGLIST copy arbitrary tags to the %s header line\n",
+ fq ? "FASTQ" : "FASTA");
+ if (fq) fprintf(to,
+" -v INT default quality score if not given in file [1]\n"
+" -i add Illumina Casava 1.8 format entry to header (eg 1:N:0:ATCACG)\n"
+" -c compression level [0..9] to use when creating gz or bgzf fastq files [1]\n"
+" --i1 FILE write first index reads to FILE\n"
+" --i2 FILE write second index reads to FILE\n"
+" --barcode-tag TAG Barcode tag [default: " DEFAULT_BARCODE_TAG "]\n"
+" --quality-tag TAG Quality tag [default: " DEFAULT_QUALITY_TAG "]\n"
+" --index-format STR How to parse barcode and quality tags\n\n");
+ sam_global_opt_help(to, "-.--.@-.");
+ fprintf(to,
+"\n"
+"The files will be automatically compressed if the file names have a .gz or .bgzf extension.\n"
+"The input to this program must be collated by name. Run 'samtools collate' or 'samtools sort -n'.\n"
+"\n"
+"Reads are designated READ1 if FLAG READ1 is set and READ2 is not set.\n"
+"Reads are designated READ2 if FLAG READ1 is not set and READ2 is set.\n"
+"Reads are designated READ_OTHER if FLAGs READ1 and READ2 are either both set\n"
+"or both unset.\n"
+"Run 'samtools flags' for more information on flag codes and meanings.\n");
+ fprintf(to,
+"\n"
+"The index-format string describes how to parse the barcode and quality tags, for example:\n"
+" i14i8 the first 14 characters are index 1, the next 8 characters are index 2\n"
+" n8i14 ignore the first 8 characters, and use the next 14 characters for index 1\n"
+"If the tag contains a separator, then the numeric part can be replaced with '*' to mean\n"
+"'read until the separator or end of tag', for example:\n"
+" n*i* ignore the left part of the tag until the separator, then use the second part\n"
+" of the tag as index 1\n");
+ fprintf(to,
+"\n"
+"Examples:\n"
+" To get just the paired reads in separate files, use:\n"
+" samtools %s -1 paired1.%s -2 paired2.%s -0 /dev/null -s /dev/null -n in.bam\n"
+"\n To get all non-supplementary/secondary reads in a single file, redirect the output:\n"
+" samtools %s in.bam > all_reads.%s\n",
+ command, fq ? "fq" : "fa", fq ? "fq" : "fa",
+ command, fq ? "fq" : "fa");
+}
+
+typedef enum { READ_UNKNOWN = 0, READ_1 = 1, READ_2 = 2 } readpart;
+typedef enum { FASTA, FASTQ } fastfile;
+typedef struct bam2fq_opts {
+ char *fnse;
+ char *fnr[3];
+ char *fn_input; // pointer to input filename in argv do not free
+ bool has12, has12always, use_oq, copy_tags, illumina_tag;
+ int flag_on, flag_off, flag_alloff;
+ sam_global_args ga;
+ fastfile filetype;
+ int def_qual;
+ char *barcode_tag;
+ char *quality_tag;
+ char *index_file[2];
+ char *index_format;
+ char *extra_tags;
+ char compression_level;
+} bam2fq_opts_t;
+
+typedef struct bam2fq_state {
+ samFile *fp;
+ BGZF *fpse;
+ BGZF *fpr[3];
+ BGZF *fpi[2];
+ BGZF *hsamtools_stdout;
+ sam_hdr_t *h;
+ bool has12, use_oq, copy_tags, illumina_tag;
+ int flag_on, flag_off, flag_alloff;
+ fastfile filetype;
+ int def_qual;
+ klist_t(ktaglist) *taglist;
+ char *index_sequence;
+ char compression_level;
+ htsThreadPool p;
+} bam2fq_state_t;
+
+/*
+ * Get and decode the read from a BAM record.
+ *
+ * TODO: htslib really needs an interface for this. Consider this or perhaps
+ * bam_get_seq_str (current vs original orientation) and bam_get_qual_str
+ * functions as string formatted equivalents to bam_get_{seq,qual}?
+ */
+
+/*
+ * Reverse a string in place.
+ * From http://stackoverflow.com/questions/8534274/is-the-strrev-function-not-available-in-linux.
+ * Author Sumit-naik: http://stackoverflow.com/users/4590926/sumit-naik
+ */
+static char *reverse(char *str)
+{
+ int i = strlen(str)-1,j=0;
+ char ch;
+ while (i>j) {
+ ch = str[i];
+ str[i]= str[j];
+ str[j] = ch;
+ i--;
+ j++;
+ }
+ return str;
+}
+
+/* return the read, reverse complemented if necessary */
+static char *get_read(const bam1_t *rec)
+{
+ int len = rec->core.l_qseq + 1;
+ char *read = calloc(1, len);
+ char *seq = (char *)bam_get_seq(rec);
+ int n;
+
+ if (!read) return NULL;
+
+ for (n=0; n < rec->core.l_qseq; n++) {
+ if (rec->core.flag & BAM_FREVERSE) read[n] = seq_nt16_str[seq_comp_table[bam_seqi(seq,n)]];
+ else read[n] = seq_nt16_str[bam_seqi(seq,n)];
+ }
+ if (rec->core.flag & BAM_FREVERSE) reverse(read);
+ return read;
+}
+
+/*
+ * get and decode the quality from a BAM record
+ */
+static int get_quality(const bam1_t *rec, char **qual_out)
+{
+ char *quality = calloc(1, rec->core.l_qseq + 1);
+ char *q = (char *)bam_get_qual(rec);
+ int n;
+
+ if (!quality) return -1;
+
+ if (*q == '\xff') {
+ free(quality);
+ *qual_out = NULL;
+ return 0;
+ }
+
+ for (n=0; n < rec->core.l_qseq; n++) {
+ quality[n] = q[n]+33;
+ }
+ if (rec->core.flag & BAM_FREVERSE) reverse(quality);
+ *qual_out = quality;
+ return 0;
+}
+
+//
+// End of htslib complaints
+//
+
+
+static readpart which_readpart(const bam1_t *b)
+{
+ if ((b->core.flag & BAM_FREAD1) && !(b->core.flag & BAM_FREAD2)) {
+ return READ_1;
+ } else if ((b->core.flag & BAM_FREAD2) && !(b->core.flag & BAM_FREAD1)) {
+ return READ_2;
+ } else {
+ return READ_UNKNOWN;
+ }
+}
+
+/*
+ * parse the length part from the index-format string
+ */
+static int getLength(char **s)
+{
+ int n = 0;
+ while (**s) {
+ if (**s == '*') { n=-1; (*s)++; break; }
+ if ( !isdigit(**s)) break;
+ n = n*10 + ((**s)-'0');
+ (*s)++;
+ }
+ return n;
+}
+
+static bool copy_tag(const char *tag, const bam1_t *rec, kstring_t *linebuf)
+{
+ uint8_t *s = bam_aux_get(rec, tag);
+ if (s) {
+ char aux_type = *s;
+ switch (aux_type) {
+ case 'C':
+ case 'S': aux_type = 'I'; break;
+ case 'c':
+ case 's': aux_type = 'i'; break;
+ case 'd': aux_type = 'f'; break;
+ }
+
+ // Ensure space. Need 6 chars + length of tag. Max length of
+ // i is 16, A is 21, B currently 26, Z is unknown, so
+ // have to check that one later.
+ if (ks_resize(linebuf, ks_len(linebuf) + 64) < 0) return false;
+
+ kputc('\t', linebuf);
+ kputsn(tag, 2, linebuf);
+ kputc(':', linebuf);
+ kputc(aux_type=='I'? 'i': aux_type, linebuf);
+ kputc(':', linebuf);
+ switch (aux_type) {
+ case 'H':
+ case 'Z':
+ if (kputs(bam_aux2Z(s), linebuf) < 0) return false;
+ break;
+ case 'i': kputw(bam_aux2i(s), linebuf); break;
+ case 'I': kputuw(bam_aux2i(s), linebuf); break;
+ case 'A': kputc(bam_aux2A(s), linebuf); break;
+ case 'f': kputd(bam_aux2f(s), linebuf); break;
+ case 'B': kputs("*** Unhandled aux type ***", linebuf); return false;
+ default: kputs("*** Unknown aux type ***", linebuf); return false;
+ }
+ }
+ return true;
+}
+
+static int insert_index_sequence_into_linebuf(char *index_sequence, kstring_t *linebuf, bam1_t *rec)
+{
+ if (!index_sequence) return 0;
+
+ kstring_t new = {0,0,NULL};
+ if (linebuf->s) {
+ char *s = strchr(linebuf->s, '\n');
+ if (s) {
+ if (ks_resize(&new, linebuf->l + strlen(index_sequence) + 16) < 0)
+ return -1;
+ *s = 0;
+ kputs(linebuf->s, &new);
+ kputc(' ', &new);
+ readpart readpart = which_readpart(rec);
+ if (readpart == READ_1) kputc('1', &new);
+ else if (readpart == READ_2) kputc('2', &new);
+ else kputc('0', &new);
+
+ kputc(':', &new);
+ if (rec->core.flag & BAM_FQCFAIL) kputc('Y', &new);
+ else kputc('N', &new);
+
+ kputs(":0:", &new);
+ kputs(index_sequence, &new);
+ kputc('\n', &new);
+ kputs(s+1, &new);
+ free(ks_release(linebuf));
+ linebuf->s = new.s; linebuf->l = new.l; linebuf->m = new.m;
+ }
+ }
+ return 0;
+}
+
+static bool make_fq_line(const bam1_t *rec, char *seq, char *qual, kstring_t *linebuf, const bam2fq_state_t *state)
+{
+ int i;
+
+ linebuf->l = 0;
+ // Write read name
+ if (kputc(state->filetype == FASTA? '>' : '@', linebuf) < 0) return false;
+ if (kputs(bam_get_qname(rec), linebuf) < 0) return false;
+ // Add the /1 /2 if requested
+ if (state->has12) {
+ readpart readpart = which_readpart(rec);
+ if (readpart == READ_1) {
+ if (kputs("/1", linebuf) < 0) return false;
+ } else if (readpart == READ_2) {
+ if (kputs("/2", linebuf) < 0) return false;
+ }
+ }
+ if (state->copy_tags) {
+ for (i = 0; copied_tags[i]; ++i) {
+ if (!copy_tag(copied_tags[i], rec, linebuf)) {
+ fprintf(samtools_stderr, "Problem copying aux tags: [%s]\n", linebuf->s);
+ return false;
+ }
+ }
+ }
+
+ if (state->taglist->size) {
+ kliter_t(ktaglist) *p;
+ for (p = kl_begin(state->taglist); p != kl_end(state->taglist); p = kl_next(p)) {
+ if (!copy_tag(kl_val(p), rec, linebuf)) {
+ fprintf(samtools_stderr, "Problem copying aux tags: [%s]\n", linebuf->s);
+ return false;
+ }
+ }
+ }
+
+ if (kputc('\n', linebuf) < 0) return false;
+ if (kputs(seq, linebuf) < 0) return false;
+ if (kputc('\n', linebuf) < 0) return false;
+
+ if (state->filetype == FASTQ) {
+ // Write quality
+ if (kputs("+\n", linebuf) < 0) return false;
+ if (qual && *qual) {
+ if (kputs(qual, linebuf) < 0) return false;
+ } else {
+ int len = strlen(seq);
+ if (ks_resize(linebuf, ks_len(linebuf) + len + 1) < 0) return false;
+ for (i = 0; i < len; ++i) {
+ kputc(33 + state->def_qual, linebuf);
+ }
+ }
+ if (kputc('\n', linebuf) < 0) return false;
+ }
+ return true;
+}
+
+/*
+ * Create FASTQ lines from the barcode tag using the index-format
+ */
+static bool tags2fq(bam1_t *rec, bam2fq_state_t *state, const bam2fq_opts_t* opts)
+{
+ uint8_t *p;
+ char *ifmt = opts->index_format;
+ char *tag = NULL;
+ char *qual = NULL;
+ char *sub_tag = NULL;
+ char *sub_qual = NULL;
+ size_t tag_len;
+ int file_number = 0;
+ kstring_t linebuf = { 0, 0, NULL }; // Buffer
+
+ if (!ifmt) return true;
+
+ // read barcode tag
+ p = bam_aux_get(rec,opts->barcode_tag);
+ if (p) tag = bam_aux2Z(p);
+
+ if (!tag) return true; // there is no tag
+
+ tag_len = strlen(tag);
+ sub_tag = calloc(1, tag_len + 1);
+ if (!sub_tag) goto fail;
+ sub_qual = calloc(1, tag_len + 1);
+ if (!sub_qual) goto fail;
+
+ // read quality tag
+ p = bam_aux_get(rec, opts->quality_tag);
+ if (p) qual = bam_aux2Z(p);
+
+ // Parse the index-format string
+ while (*ifmt) {
+ if (file_number > 1) break; // shouldn't happen if we've validated paramaters correctly
+ char action = *ifmt; // should be 'i' or 'n'
+ ifmt++; // skip over action
+ int index_len = getLength(&ifmt);
+ int n = 0;
+
+ if (index_len < 0) {
+ // read until separator
+ while (isalpha(*tag)) {
+ sub_tag[n] = *tag++;
+ if (qual) sub_qual[n] = *qual++;
+ n++;
+ }
+ if (*tag) { // skip separator
+ tag++;
+ if (qual) qual++;
+ }
+ } else {
+ // read index_len characters
+ while (index_len-- && *tag) {
+ sub_tag[n] = *tag++;
+ if (qual) sub_qual[n] = *qual++;
+ n++;
+ }
+ }
+ sub_tag[n] = '\0';
+ sub_qual[n] = '\0';
+
+ if (action=='i' && *sub_tag) {
+ if (state->index_sequence) {
+ char *new_index_sequence = realloc(state->index_sequence, strlen(state->index_sequence) + strlen(sub_tag) + 2);
+ if (!new_index_sequence) goto fail;
+ state->index_sequence = new_index_sequence;
+ strcat(state->index_sequence, INDEX_SEPARATOR);
+ strcat(state->index_sequence, sub_tag);
+ } else {
+ state->index_sequence = strdup(sub_tag); // we're going to need this later...
+ }
+ if (!state->index_sequence) goto fail;
+ if (!make_fq_line(rec, sub_tag, sub_qual, &linebuf, state)) goto fail;
+ if (state->illumina_tag) {
+ if (insert_index_sequence_into_linebuf(sub_tag, &linebuf, rec) < 0) {
+ goto fail;
+ }
+ }
+ if (state->fpi[file_number]) {
+ if (bgzf_write(state->fpi[file_number++], linebuf.s, linebuf.l) < 0)
+ goto fail;
+ }
+ }
+
+ }
+
+ free(sub_qual); free(sub_tag);
+ free(linebuf.s);
+ return true;
+
+ fail:
+ perror(__func__);
+ free(sub_qual); free(sub_tag);
+ free(linebuf.s);
+ return false;
+}
+
+// Transform a bam1_t record into a string with the FASTQ representation of it
+// @returns false for error, true for success
+static bool bam1_to_fq(const bam1_t *b, kstring_t *linebuf, const bam2fq_state_t *state)
+{
+ int32_t qlen = b->core.l_qseq;
+ assert(qlen >= 0);
+ const uint8_t *oq = NULL;
+ char *qual = NULL;
+
+ char *seq = get_read(b);
+ if (!seq) return false;
+
+ if (state->use_oq) oq = bam_aux_get(b, "OQ");
+ if (oq && *oq=='Z') {
+ qual = strdup(bam_aux2Z(oq));
+ if (!qual) goto fail;
+ if (b->core.flag & BAM_FREVERSE) { // read is reverse complemented
+ reverse(qual);
+ }
+ } else {
+ if (get_quality(b, &qual) < 0) goto fail;
+ }
+
+ if (!make_fq_line(b, seq, qual, linebuf, state)) goto fail;
+
+ free(qual);
+ free(seq);
+ return true;
+
+ fail:
+ free(seq);
+ free(qual);
+ return false;
+}
+
+static void free_opts(bam2fq_opts_t *opts)
+{
+ free(opts->barcode_tag);
+ free(opts->quality_tag);
+ free(opts->index_format);
+ free(opts->extra_tags);
+ free(opts);
+}
+
+// return true if valid
+static bool parse_opts(int argc, char *argv[], bam2fq_opts_t** opts_out)
+{
+ // Parse args
+ bam2fq_opts_t* opts = calloc(1, sizeof(bam2fq_opts_t));
+ opts->has12 = true;
+ opts->has12always = false;
+ opts->filetype = FASTQ;
+ opts->def_qual = 1;
+ opts->barcode_tag = NULL;
+ opts->quality_tag = NULL;
+ opts->index_format = NULL;
+ opts->index_file[0] = NULL;
+ opts->index_file[1] = NULL;
+ opts->extra_tags = NULL;
+ opts->compression_level = 1;
+ opts->flag_off = BAM_FSECONDARY|BAM_FSUPPLEMENTARY;
+ int flag_off_set = 0;
+
+ int c;
+ sam_global_args_init(&opts->ga);
+ static const struct option lopts[] = {
+ SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0, '@'),
+ {"i1", required_argument, NULL, 1},
+ {"I1", required_argument, NULL, 1},
+ {"i2", required_argument, NULL, 2},
+ {"I2", required_argument, NULL, 2},
+ {"if", required_argument, NULL, 3},
+ {"IF", required_argument, NULL, 3},
+ {"index-format", required_argument, NULL, 3},
+ {"barcode-tag", required_argument, NULL, 'b'},
+ {"quality-tag", required_argument, NULL, 'q'},
+ { NULL, 0, NULL, 0 }
+ };
+ while ((c = getopt_long(argc, argv, "0:1:2:o:f:F:G:niNOs:c:tT:v:@:", lopts, NULL)) > 0) {
+ switch (c) {
+ case 'b': opts->barcode_tag = strdup(optarg); break;
+ case 'q': opts->quality_tag = strdup(optarg); break;
+ case 1 : opts->index_file[0] = optarg; break;
+ case 2 : opts->index_file[1] = optarg; break;
+ case 3 : opts->index_format = strdup(optarg); break;
+ case '0': opts->fnr[0] = optarg; break;
+ case '1': opts->fnr[1] = optarg; break;
+ case '2': opts->fnr[2] = optarg; break;
+ case 'o': opts->fnr[1] = optarg; opts->fnr[2] = optarg; break;
+ case 'f': opts->flag_on |= strtol(optarg, 0, 0); break;
+ case 'F':
+ if (!flag_off_set) {
+ flag_off_set = 1;
+ opts->flag_off = 0;
+ }
+ opts->flag_off |= strtol(optarg, 0, 0); break;
+ case 'G': opts->flag_alloff |= strtol(optarg, 0, 0); break;
+ case 'n': opts->has12 = false; break;
+ case 'N': opts->has12always = true; break;
+ case 'O': opts->use_oq = true; break;
+ case 's': opts->fnse = optarg; break;
+ case 't': opts->copy_tags = true; break;
+ case 'i': opts->illumina_tag = true; break;
+ case 'c': opts->compression_level = atoi(optarg); break;
+ case 'T': opts->extra_tags = strdup(optarg); break;
+ case 'v': opts->def_qual = atoi(optarg); break;
+ case '?': bam2fq_usage(samtools_stderr, argv[0]); free_opts(opts); return false;
+ default:
+ if (parse_sam_global_opt(c, optarg, lopts, &opts->ga) != 0) {
+ bam2fq_usage(samtools_stderr, argv[0]); free_opts(opts); return false;
+ }
+ break;
+ }
+ }
+
+ if (opts->fnr[1] || opts->fnr[2]) opts->has12 = false;
+ if (opts->has12always) opts->has12 = true;
+
+ if (!opts->barcode_tag) opts->barcode_tag = strdup(DEFAULT_BARCODE_TAG);
+ if (!opts->quality_tag) opts->quality_tag = strdup(DEFAULT_QUALITY_TAG);
+
+ int nIndex = 0;
+ if (opts->index_format) {
+ char *s;
+ for (s = opts->index_format; *s; s++) {
+ if (*s == 'i') nIndex++;
+ }
+ }
+ if (nIndex>2) {
+ fprintf(samtools_stderr,"Invalid index format: more than 2 indexes\n");
+ bam2fq_usage(samtools_stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ if (opts->index_file[1] && !opts->index_file[0]) {
+ fprintf(samtools_stderr, "Index one specified, but index two not given\n");
+ bam2fq_usage(samtools_stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ if (opts->illumina_tag && !nIndex) {
+ fprintf(samtools_stderr, "You must specify an index format (--index-format) with the Illumina Casava (-i) option\n");
+ bam2fq_usage(samtools_stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ if (nIndex==0 && opts->index_file[0]) {
+ fprintf(samtools_stderr, "index_format not specified, but index file given\n");
+ bam2fq_usage(samtools_stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ if (opts->def_qual < 0 || 93 < opts->def_qual) {
+ fprintf(samtools_stderr, "Invalid -v default quality %i, allowed range 0 to 93\n", opts->def_qual);
+ bam2fq_usage(samtools_stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ const char* type_str = argv[0];
+ if (strcasecmp("fastq", type_str) == 0 || strcasecmp("bam2fq", type_str) == 0) {
+ opts->filetype = FASTQ;
+ } else if (strcasecmp("fasta", type_str) == 0) {
+ opts->filetype = FASTA;
+ } else {
+ print_error("bam2fq", "Unrecognised type call \"%s\", this should be impossible... but you managed it!", type_str);
+ bam2fq_usage(samtools_stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+
+ if (argc == optind && isatty(STDIN_FILENO)) {
+ bam2fq_usage(samtools_stdout, argv[0]);
+ free_opts(opts);
+ return true;
+ }
+
+ if (argc - optind > 1) {
+ fprintf(samtools_stderr, "Too many arguments.\n");
+ bam2fq_usage(samtools_stderr, argv[0]);
+ free_opts(opts);
+ return false;
+ }
+ opts->fn_input = argc > optind ? argv[optind] : "-";
+ *opts_out = opts;
+ return true;
+}
+
+static BGZF *open_fqfile(char *filename, int c, htsThreadPool *tp)
+{
+ char mode[4] = "w";
+ size_t len = strlen(filename);
+
+ mode[2] = 0; mode[3] = 0;
+ if (len > 3 && strstr(filename + (len - 3),".gz")) {
+ mode[1] = 'g'; mode[2] = c+'0';
+ } else if ((len > 4 && strstr(filename + (len - 4),".bgz"))
+ || (len > 5 && strstr(filename + (len - 5),".bgzf"))) {
+ mode[1] = c+'0';
+ } else {
+ mode[1] = 'u';
+ }
+
+ BGZF *fp = bgzf_open(filename,mode);
+ if (!fp)
+ return fp;
+ if (tp->pool && bgzf_thread_pool(fp, tp->pool, tp->qsize) < 0) {
+ bgzf_close(fp);
+ return NULL;
+ }
+ return fp;
+}
+
+static bool init_state(const bam2fq_opts_t* opts, bam2fq_state_t** state_out)
+{
+ bam2fq_state_t* state = calloc(1, sizeof(bam2fq_state_t));
+ state->flag_on = opts->flag_on;
+ state->flag_off = opts->flag_off;
+ state->flag_alloff = opts->flag_alloff;
+ state->has12 = opts->has12;
+ state->use_oq = opts->use_oq;
+ state->illumina_tag = opts->illumina_tag;
+ state->copy_tags = opts->copy_tags;
+ state->filetype = opts->filetype;
+ state->def_qual = opts->def_qual;
+ state->index_sequence = NULL;
+ state->hsamtools_stdout = NULL;
+ state->compression_level = opts->compression_level;
+
+ state->taglist = kl_init(ktaglist);
+ if (opts->extra_tags) {
+ char *save_p;
+ char *s = strtok_r(opts->extra_tags, ",", &save_p);
+ while (s) {
+ if (strlen(s) != 2) {
+ fprintf(samtools_stderr, "Parsing extra tags - '%s' is not two characters\n", s);
+ free(state);
+ return false;
+ }
+ char **et = kl_pushp(ktaglist, state->taglist);
+ *et = s;
+ s = strtok_r(NULL, ",", &save_p);
+ }
+ }
+
+ state->fp = sam_open(opts->fn_input, "r");
+ if (state->fp == NULL) {
+ print_error_errno("bam2fq","Cannot read file \"%s\"", opts->fn_input);
+ free(state);
+ return false;
+ }
+
+ state->p.pool = NULL;
+ if (opts->ga.nthreads > 0) {
+ if (!(state->p.pool = hts_tpool_init(opts->ga.nthreads))) {
+ fprintf(samtools_stderr, "Failed to create thread pool\n");
+ free(state);
+ return false;
+ }
+ state->p.qsize = opts->ga.nthreads*2;
+ hts_set_thread_pool(state->fp, &state->p);
+ }
+
+ uint32_t rf = SAM_QNAME | SAM_FLAG | SAM_SEQ | SAM_QUAL;
+ if (opts->use_oq || opts->extra_tags || opts->index_file[0]) rf |= SAM_AUX;
+ if (hts_set_opt(state->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
+ fprintf(samtools_stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
+ free(state);
+ return false;
+ }
+ if (hts_set_opt(state->fp, CRAM_OPT_DECODE_MD, 0)) {
+ fprintf(samtools_stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
+ free(state);
+ return false;
+ }
+ if (opts->fnse) {
+ state->fpse = open_fqfile(opts->fnse, state->compression_level, &state->p);
+ if (state->fpse == NULL) {
+ print_error_errno("bam2fq", "Cannot write to singleton file \"%s\"", opts->fnse);
+ free(state);
+ return false;
+ }
+ }
+
+ if (opts->ga.reference) {
+ if (hts_set_fai_filename(state->fp, opts->ga.reference) != 0) {
+ print_error_errno("bam2fq", "cannot load reference \"%s\"", opts->ga.reference);
+ free(state);
+ return false;
+ }
+ }
+
+ int i, j;
+ for (i = 0; i < 3; ++i) {
+ if (opts->fnr[i]) {
+ for (j = 0; j < i; j++)
+ if (opts->fnr[j] && strcmp(opts->fnr[j], opts->fnr[i]) == 0)
+ break;
+ if (j == i) {
+ state->fpr[i] = open_fqfile(opts->fnr[i], state->compression_level, &state->p);
+ if (state->fpr[i] == NULL) {
+ print_error_errno("bam2fq", "Cannot write to r%d file \"%s\"",
+ i, opts->fnr[i]);
+ free(state);
+ return false;
+ }
+ } else {
+ state->fpr[i] = state->fpr[j];
+ }
+ } else {
+ if (!state->hsamtools_stdout) {
+ state->hsamtools_stdout = bgzf_dopen(fileno(samtools_stdout), "wu");
+ if (!state->hsamtools_stdout) {
+ print_error_errno("bam2fq", "Cannot open STDOUT");
+ free(state);
+ return false;
+ }
+ }
+ state->fpr[i] = state->hsamtools_stdout;
+ }
+ }
+ for (i = 0; i < 2; i++) {
+ state->fpi[i] = NULL;
+ if (opts->index_file[i]) {
+ for (j = 0; j < 3; j++)
+ if (opts->fnr[j] && strcmp(opts->fnr[j], opts->index_file[i]) == 0)
+ break;
+ for (j -= 3; j >= 0 && j < i; j++)
+ if (opts->index_file[j] && strcmp(opts->index_file[j], opts->index_file[i]) == 0)
+ break;
+ if (i == j) {
+ state->fpi[i] = open_fqfile(opts->index_file[i], state->compression_level, &state->p);
+ if (state->fpi[i] == NULL) {
+ print_error_errno("bam2fq", "Cannot write to i%d file \"%s\"",
+ i+1, opts->index_file[i]);
+ free(state);
+ return false;
+ }
+ } else if (j < 0) {
+ state->fpi[i] = state->fpr[j+3];
+ } else {
+ state->fpi[i] = state->fpi[j];
+ }
+ }
+ }
+
+ state->h = sam_hdr_read(state->fp);
+ if (state->h == NULL) {
+ fprintf(samtools_stderr, "Failed to read header for \"%s\"\n", opts->fn_input);
+ free(state);
+ return false;
+ }
+
+ *state_out = state;
+ return true;
+}
+
+static bool destroy_state(const bam2fq_opts_t *opts, bam2fq_state_t *state, int* status)
+{
+ bool valid = true;
+ sam_hdr_destroy(state->h);
+ check_sam_close("bam2fq", state->fp, opts->fn_input, "file", status);
+ if (state->fpse && bgzf_close(state->fpse)) { print_error_errno("bam2fq", "Error closing singleton file \"%s\"", opts->fnse); valid = false; }
+ int i, j;
+ for (i = 0; i < 3; ++i) {
+ if (state->fpr[i] != state->hsamtools_stdout) {
+ for (j = 0; j < i; j++)
+ if (state->fpr[i] == state->fpr[j])
+ break;
+ if (j == i && bgzf_close(state->fpr[i])) {
+ print_error_errno("bam2fq", "Error closing r%d file \"%s\"", i, opts->fnr[i]);
+ valid = false;
+ }
+ }
+ }
+ if (state->hsamtools_stdout) {
+ if (bgzf_close(state->hsamtools_stdout)) {
+ print_error_errno("bam2fq", "Error closing STDOUT");
+ valid = false;
+ }
+ }
+ for (i = 0; i < 2; i++) {
+ for (j = 0; j < 3; j++)
+ if (state->fpi[i] == state->fpr[j])
+ break;
+ for (j -= 3; j >= 0 && j < i; j++)
+ if (state->fpi[i] == state->fpi[j])
+ break;
+ if (j == i && state->fpi[i] && bgzf_close(state->fpi[i])) {
+ print_error_errno("bam2fq", "Error closing i%d file \"%s\"", i+1, opts->index_file[i]);
+ valid = false;
+ }
+ }
+ kl_destroy(ktaglist,state->taglist);
+ free(state->index_sequence);
+ if (state->p.pool)
+ hts_tpool_destroy(state->p.pool);
+ free(state);
+ return valid;
+}
+
+static inline bool filter_it_out(const bam1_t *b, const bam2fq_state_t *state)
+{
+ return ((b->core.flag&(state->flag_on)) != state->flag_on // or reads indicated by filter flags
+ || (b->core.flag&(state->flag_off)) != 0
+ || (b->core.flag&(state->flag_alloff) && (b->core.flag&(state->flag_alloff)) == state->flag_alloff));
+
+}
+
+static bool bam2fq_mainloop(bam2fq_state_t *state, bam2fq_opts_t* opts)
+{
+ int n;
+ bam1_t *records[3] = {NULL, NULL, NULL};
+ char *current_qname = NULL;
+ int64_t n_reads = 0, n_singletons = 0; // Statistics
+ kstring_t linebuf[3] = {{0,0,NULL},{0,0,NULL},{0,0,NULL}};
+ int score[3];
+ int at_eof;
+ bool valid = true;
+ bam1_t* b = NULL;
+
+ while (true) {
+ if (!b)
+ b = bam_init1();
+ if (b == NULL) {
+ perror("[bam2fq_mainloop] Malloc error for bam record buffer.");
+ valid = false;
+ break;
+ }
+ int res = sam_read1(state->fp, state->h, b);
+ if (res < -1) {
+ fprintf(samtools_stderr, "[bam2fq_mainloop] Failed to read bam record.\n");
+ valid = false;
+ break;
+ }
+ at_eof = res < 0;
+
+ if (!at_eof && filter_it_out(b, state))
+ continue;
+ if (!at_eof) ++n_reads;
+
+ if (at_eof || !current_qname || (strcmp(current_qname, bam_get_qname(b)) != 0)) {
+ if (current_qname) {
+ if (state->illumina_tag) {
+ for (n=0; valid && n<3; n++) {
+ if (!records[n]) continue;
+ if (insert_index_sequence_into_linebuf(state->index_sequence, &linebuf[n], records[n]) < 0) valid = false;
+ }
+ if (!valid) break;
+ }
+ free(state->index_sequence); state->index_sequence = NULL;
+ if (score[1] > 0 && score[2] > 0) {
+ // print linebuf[1] to fpr[1], linebuf[2] to fpr[2]
+ if (bgzf_write(state->fpr[1], linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
+ if (bgzf_write(state->fpr[2], linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
+ } else if (score[1] > 0 || score[2] > 0) {
+ if (state->fpse) {
+ // print whichever one exists to fpse
+ if (score[1] > 0) {
+ if (bgzf_write(state->fpse, linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
+ } else {
+ if (bgzf_write(state->fpse, linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
+ }
+ ++n_singletons;
+ } else {
+ if (score[1] > 0) {
+ if (bgzf_write(state->fpr[1], linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
+ } else {
+ if (bgzf_write(state->fpr[2], linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
+ }
+ }
+ }
+ if (score[0]) { // TODO: check this
+ // print linebuf[0] to fpr[0]
+ if (bgzf_write(state->fpr[0], linebuf[0].s, linebuf[0].l) < 0) { valid = false; break; }
+ }
+ }
+
+
+ free(current_qname); current_qname = NULL;
+ score[0] = score[1] = score[2] = 0;
+ for (n=0; n < 3; n++) {
+ bam_destroy1(records[n]); records[n]=NULL;
+ }
+
+ if (at_eof) { break; }
+
+ current_qname = strdup(bam_get_qname(b));
+ if (!current_qname) { valid = false; break; }
+ }
+
+ // Prefer a copy of the read that has base qualities
+ int b_score = bam_get_qual(b)[0] != 0xff? 2 : 1;
+ readpart rp = which_readpart(b);
+ if (b_score > score[rp]) {
+ if (!tags2fq(b, state, opts)) { valid = false; break; }
+ if (records[rp]) bam_destroy1(records[rp]);
+ records[rp] = b;
+ score[rp] = b_score;
+ b = NULL;
+ if(!bam1_to_fq(records[rp], &linebuf[rp], state)) {
+ fprintf(samtools_stderr, "[%s] Error converting read to FASTA/Q\n", __func__);
+ valid = false; break;
+ }
+ }
+ }
+ if (!valid)
+ {
+ perror("[bam2fq_mainloop] Error writing to FASTx files.");
+ }
+ bam_destroy1(b);
+ for (n=0; n < 3; n++) {
+ bam_destroy1(records[n]);
+ }
+ free(current_qname);
+ free(linebuf[0].s);
+ free(linebuf[1].s);
+ free(linebuf[2].s);
+ fprintf(samtools_stderr, "[M::%s] discarded %" PRId64 " singletons\n", __func__, n_singletons);
+ fprintf(samtools_stderr, "[M::%s] processed %" PRId64 " reads\n", __func__, n_reads);
+
+ return valid;
+}
+
+int main_bam2fq(int argc, char *argv[])
+{
+ int status = EXIT_SUCCESS;
+ bam2fq_opts_t* opts = NULL;
+ bam2fq_state_t* state = NULL;
+
+ bool valid = parse_opts(argc, argv, &opts);
+ if (!valid || opts == NULL) return valid ? EXIT_SUCCESS : EXIT_FAILURE;
+
+ if (!init_state(opts, &state)) return EXIT_FAILURE;
+
+ if (!bam2fq_mainloop(state,opts)) status = EXIT_FAILURE;
+
+ if (!destroy_state(opts, state, &status)) return EXIT_FAILURE;
+ sam_global_args_free(&opts->ga);
+ free_opts(opts);
+
+ return status;
+}
+++ /dev/null
-/* bam_import.c -- SAM format parsing.
-
- Copyright (C) 2008-2013 Genome Research Ltd.
-
- Author: Heng Li <lh3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include <zlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-#include "htslib/kstring.h"
-#include "bam.h"
-#include "htslib/kseq.h"
-
-KSTREAM_INIT(gzFile, gzread, 16384)
-
-bam_header_t *sam_header_read2(const char *fn)
-{
- bam_header_t *header;
- int c, dret, n_targets = 0;
- gzFile fp;
- kstream_t *ks;
- kstring_t *str;
- kstring_t samstr = { 0, 0, NULL };
- if (fn == 0) return 0;
- fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r");
- if (fp == 0) return 0;
- ks = ks_init(fp);
- str = (kstring_t*)calloc(1, sizeof(kstring_t));
- while (ks_getuntil(ks, 0, str, &dret) > 0) {
- ksprintf(&samstr, "@SQ\tSN:%s", str->s);
- ks_getuntil(ks, 0, str, &dret);
- ksprintf(&samstr, "\tLN:%d\n", atoi(str->s));
- n_targets++;
- if (dret != '\n')
- while ((c = ks_getc(ks)) != '\n' && c != -1);
- }
- ks_destroy(ks);
- gzclose(fp);
- free(str->s); free(str);
- header = sam_hdr_parse(samstr.l, samstr.s? samstr.s : "");
- free(samstr.s);
- fprintf(stderr, "[sam_header_read2] %d sequences loaded.\n", n_targets);
- return header;
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* bam_import.c -- SAM format parsing.
-
- Copyright (C) 2008-2013 Genome Research Ltd.
-
- Author: Heng Li <lh3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include <zlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-#include "htslib/kstring.h"
-#include "bam.h"
-#include "htslib/kseq.h"
-
-KSTREAM_INIT(gzFile, gzread, 16384)
-
-bam_header_t *sam_header_read2(const char *fn)
-{
- bam_header_t *header;
- int c, dret, n_targets = 0;
- gzFile fp;
- kstream_t *ks;
- kstring_t *str;
- kstring_t samstr = { 0, 0, NULL };
- if (fn == 0) return 0;
- fp = (strcmp(fn, "-") == 0)? gzdopen(fileno(stdin), "r") : gzopen(fn, "r");
- if (fp == 0) return 0;
- ks = ks_init(fp);
- str = (kstring_t*)calloc(1, sizeof(kstring_t));
- while (ks_getuntil(ks, 0, str, &dret) > 0) {
- ksprintf(&samstr, "@SQ\tSN:%s", str->s);
- ks_getuntil(ks, 0, str, &dret);
- ksprintf(&samstr, "\tLN:%d\n", atoi(str->s));
- n_targets++;
- if (dret != '\n')
- while ((c = ks_getc(ks)) != '\n' && c != -1);
- }
- ks_destroy(ks);
- gzclose(fp);
- free(str->s); free(str);
- header = sam_hdr_parse(samstr.l, samstr.s? samstr.s : "");
- free(samstr.s);
- fprintf(samtools_stderr, "[sam_header_read2] %d sequences loaded.\n", n_targets);
- return header;
-}
/* bam_index.c -- index and idxstats subcommands.
- Copyright (C) 2008-2011, 2013, 2014 Genome Research Ltd.
+ Copyright (C) 2008-2011, 2013-2016, 2018, 2019 Genome Research Ltd.
Portions copyright (C) 2010 Broad Institute.
Portions copyright (C) 2013 Peter Cock, The James Hutton Institute.
* Returns 0 on success,
* -1 on failure.
*/
-int slow_idxstats(samFile *fp, bam_hdr_t *header) {
+int slow_idxstats(samFile *fp, sam_hdr_t *header) {
int ret, last_tid = -2;
bam1_t *b = bam_init1();
if (hts_set_opt(fp, CRAM_OPT_REQUIRED_FIELDS, SAM_RNAME | SAM_FLAG))
return -1;
- uint64_t (*count0)[2] = calloc(header->n_targets+1, sizeof(*count0));
+ uint64_t (*count0)[2] = calloc(sam_hdr_nref(header)+1, sizeof(*count0));
uint64_t (*counts)[2] = count0+1;
if (!count0)
return -1;
while ((ret = sam_read1(fp, header, b)) >= 0) {
- if (b->core.tid >= header->n_targets || b->core.tid < -1) {
+ if (b->core.tid >= sam_hdr_nref(header) || b->core.tid < -1) {
free(count0);
return -1;
}
if (ret == -1) {
int i;
- for (i = 0; i < header->n_targets; i++) {
- printf("%s\t%d\t%"PRIu64"\t%"PRIu64"\n",
- header->target_name[i],
- header->target_len[i],
+ for (i = 0; i < sam_hdr_nref(header); i++) {
+ printf("%s\t%"PRId64"\t%"PRIu64"\t%"PRIu64"\n",
+ sam_hdr_tid2name(header, i),
+ (int64_t) sam_hdr_tid2len(header, i),
counts[i][0], counts[i][1]);
}
printf("*\t0\t%"PRIu64"\t%"PRIu64"\n", counts[-1][0], counts[-1][1]);
static void usage_exit(FILE *fp, int exit_status)
{
fprintf(fp, "Usage: samtools idxstats [options] <in.bam>\n");
- sam_global_opt_help(fp, "-.---@");
+ sam_global_opt_help(fp, "-.---@-.");
exit(exit_status);
}
int bam_idxstats(int argc, char *argv[])
{
hts_idx_t* idx;
- bam_hdr_t* header;
+ sam_hdr_t* header;
samFile* fp;
int c;
}
int i;
- for (i = 0; i < header->n_targets; ++i) {
+ for (i = 0; i < sam_hdr_nref(header); ++i) {
// Print out contig name and length
- printf("%s\t%d", header->target_name[i], header->target_len[i]);
+ printf("%s\t%"PRId64, sam_hdr_tid2name(header, i), (int64_t) sam_hdr_tid2len(header, i));
// Now fetch info about it from the meta bin
uint64_t u, v;
hts_idx_get_stat(idx, i, &u, &v);
hts_idx_destroy(idx);
}
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
sam_close(fp);
return 0;
}
/* bam_index.c -- index and idxstats subcommands.
- Copyright (C) 2008-2011, 2013, 2014 Genome Research Ltd.
+ Copyright (C) 2008-2011, 2013-2016, 2018, 2019 Genome Research Ltd.
Portions copyright (C) 2010 Broad Institute.
Portions copyright (C) 2013 Peter Cock, The James Hutton Institute.
* Returns 0 on success,
* -1 on failure.
*/
-int slow_idxstats(samFile *fp, bam_hdr_t *header) {
+int slow_idxstats(samFile *fp, sam_hdr_t *header) {
int ret, last_tid = -2;
bam1_t *b = bam_init1();
if (hts_set_opt(fp, CRAM_OPT_REQUIRED_FIELDS, SAM_RNAME | SAM_FLAG))
return -1;
- uint64_t (*count0)[2] = calloc(header->n_targets+1, sizeof(*count0));
+ uint64_t (*count0)[2] = calloc(sam_hdr_nref(header)+1, sizeof(*count0));
uint64_t (*counts)[2] = count0+1;
if (!count0)
return -1;
while ((ret = sam_read1(fp, header, b)) >= 0) {
- if (b->core.tid >= header->n_targets || b->core.tid < -1) {
+ if (b->core.tid >= sam_hdr_nref(header) || b->core.tid < -1) {
free(count0);
return -1;
}
if (ret == -1) {
int i;
- for (i = 0; i < header->n_targets; i++) {
- fprintf(samtools_stdout, "%s\t%d\t%"PRIu64"\t%"PRIu64"\n",
- header->target_name[i],
- header->target_len[i],
+ for (i = 0; i < sam_hdr_nref(header); i++) {
+ fprintf(samtools_stdout, "%s\t%"PRId64"\t%"PRIu64"\t%"PRIu64"\n",
+ sam_hdr_tid2name(header, i),
+ (int64_t) sam_hdr_tid2len(header, i),
counts[i][0], counts[i][1]);
}
fprintf(samtools_stdout, "*\t0\t%"PRIu64"\t%"PRIu64"\n", counts[-1][0], counts[-1][1]);
static void usage_exit(FILE *fp, int exit_status)
{
fprintf(fp, "Usage: samtools idxstats [options] <in.bam>\n");
- sam_global_opt_help(fp, "-.---@");
+ sam_global_opt_help(fp, "-.---@-.");
exit(exit_status);
}
int bam_idxstats(int argc, char *argv[])
{
hts_idx_t* idx;
- bam_hdr_t* header;
+ sam_hdr_t* header;
samFile* fp;
int c;
}
int i;
- for (i = 0; i < header->n_targets; ++i) {
+ for (i = 0; i < sam_hdr_nref(header); ++i) {
// Print out contig name and length
- fprintf(samtools_stdout, "%s\t%d", header->target_name[i], header->target_len[i]);
+ fprintf(samtools_stdout, "%s\t%"PRId64, sam_hdr_tid2name(header, i), (int64_t) sam_hdr_tid2len(header, i));
// Now fetch info about it from the meta bin
uint64_t u, v;
hts_idx_get_stat(idx, i, &u, &v);
hts_idx_destroy(idx);
}
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
sam_close(fp);
return 0;
}
buf->n_nodes = 0;
}
-static int tview_func(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data)
+static int tview_func(uint32_t tid, hts_pos_t pos, int n, const bam_pileup1_t *pl, void *data)
{
bam_lplbuf_t *tv = (bam_lplbuf_t*)data;
freenode_t *p;
buf->n_nodes = 0;
}
-static int tview_func(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data)
+static int tview_func(uint32_t tid, hts_pos_t pos, int n, const bam_pileup1_t *pl, void *data)
{
bam_lplbuf_t *tv = (bam_lplbuf_t*)data;
freenode_t *p;
#ifndef BAM_PILEUP_F_DEFINED
#define BAM_PILEUP_F_DEFINED
-typedef int (*bam_pileup_f)(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data);
+typedef int (*bam_pileup_f)(uint32_t tid, hts_pos_t pos, int n, const bam_pileup1_t *pl, void *data);
#endif //BAM_PILEUP_F_DEFINED
/* bam_markdup.c -- Mark duplicates from a coord sorted file that has gone
through fixmates with the mate scoring option on.
- Copyright (C) 2017-18 Genome Research Ltd.
+ Copyright (C) 2017-2019 Genome Research Ltd.
Author: Andrew Whitwham <aw7@sanger.ac.uk>
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE
+
+Estimate library size derived from Picard DuplicationMetrics.java
+Copyright (c) 2009,2018 The Broad Institute. MIT license.
*/
#include <config.h>
#include <ctype.h>
#include <time.h>
#include <sys/stat.h>
+#include <math.h>
#include "htslib/thread_pool.h"
#include "htslib/sam.h"
#include "sam_opts.h"
#include "htslib/kstring.h"
#include "tmp_file.h"
+
+typedef struct {
+ samFile *in;
+ samFile *out;
+ char *prefix;
+ int remove_dups;
+ int32_t max_length;
+ int do_stats;
+ int supp;
+ int tag;
+ int opt_dist;
+ int no_pg;
+ int clear;
+ int mode;
+ int write_index;
+ int include_fails;
+ char *stats_file;
+ char *arg_list;
+ char *out_fn;
+} md_param_t;
+
typedef struct {
- int32_t single;
+ hts_pos_t this_coord;
+ hts_pos_t other_coord;
int32_t this_ref;
- int32_t this_coord;
int32_t other_ref;
- int32_t other_coord;
- int32_t leftmost;
- int32_t orientation;
+ int8_t single;
+ int8_t leftmost;
+ int8_t orientation;
} key_data_t;
+typedef struct read_queue_s {
+ key_data_t pair_key;
+ key_data_t single_key;
+ bam1_t *b;
+ struct read_queue_s *duplicate;
+ hts_pos_t pos;
+} read_queue_t;
+
typedef struct {
- bam1_t *p;
+ read_queue_t *p;
} in_hash_t;
typedef struct {
- bam1_t *b;
- int32_t pos;
- key_data_t pair_key;
- key_data_t single_key;
-} read_queue_t;
+ char *name;
+ char type;
+} dup_map_t;
khint_t hash;
if (key.single) {
- unsigned char sig[12];
+ unsigned char sig[13];
memcpy(sig + i, &key.this_ref, 4); i += 4;
- memcpy(sig + i, &key.this_coord, 4); i += 4;
- memcpy(sig + i, &key.orientation, 4); i += 4;
+ memcpy(sig + i, &key.this_coord, 8); i += 8;
+ memcpy(sig + i, &key.orientation, 1); i += 1;
hash = do_hash(sig, i);
} else {
- unsigned char sig[24];
+ unsigned char sig[26];
memcpy(sig + i, &key.this_ref, 4); i += 4;
- memcpy(sig + i, &key.this_coord, 4); i += 4;
+ memcpy(sig + i, &key.this_coord, 8); i += 8;
memcpy(sig + i, &key.other_ref, 4); i += 4;
- memcpy(sig + i, &key.other_coord, 4); i += 4;
- memcpy(sig + i, &key.leftmost, 4); i += 4;
- memcpy(sig + i, &key.orientation, 4); i += 4;
+ memcpy(sig + i, &key.other_coord, 8); i += 8;
+ memcpy(sig + i, &key.leftmost, 1); i += 1;
+ memcpy(sig + i, &key.orientation, 1); i += 1;
hash = do_hash(sig, i);
}
#define __free_queue_element(p)
+
+// Orientations (prime numbers to feed to hashing algorithm)
#define O_FF 2
#define O_RR 3
#define O_FR 5
#define O_RF 7
+// Left or rightmost
+#define R_LE 11
+#define R_RI 13
+
+#define BMD_WARNING_MAX 10
+
+#define MD_MIN_QUALITY 15
+
+// Duplicate finding mode
+#define MD_MODE_TEMPLATE 0
+#define MD_MODE_SEQUENCE 1
+
KHASH_INIT(reads, key_data_t, in_hash_t, 1, hash_key, key_equal) // read map hash
KLIST_INIT(read_queue, read_queue_t, __free_queue_element) // the reads buffer
-KHASH_MAP_INIT_STR(duplicates, int) // map of duplicates for supplementary dup id
+KHASH_MAP_INIT_STR(duplicates, dup_map_t) // map of duplicates for supplementary dup id
/* Calculate the mate's unclipped start based on position and cigar string from MC tag. */
-static int32_t unclipped_other_start(int32_t op, char *cigar) {
+static hts_pos_t unclipped_other_start(hts_pos_t op, char *cigar) {
char *c = cigar;
- int32_t clipped = 0;
+ int64_t clipped = 0;
while (*c && *c != '*') {
long num = 0;
/* Calculate the current read's start based on the stored cigar string. */
-static int32_t unclipped_start(bam1_t *b) {
+static hts_pos_t unclipped_start(bam1_t *b) {
uint32_t *cigar = bam_get_cigar(b);
- int32_t clipped = 0;
+ int64_t clipped = 0;
uint32_t i;
for (i = 0; i < b->core.n_cigar; i++) {
/* Calculate the mate's unclipped end based on start position and cigar string from MC tag.*/
-static int32_t unclipped_other_end(int32_t op, char *cigar) {
+static hts_pos_t unclipped_other_end(int64_t op, char *cigar) {
char *c = cigar;
- int32_t refpos = 0;
+ int64_t refpos = 0;
int skip = 1;
while (*c && *c != '*') {
/* Calculate the current read's end based on the stored cigar string. */
-static int32_t unclipped_end(bam1_t *b) {
+static hts_pos_t unclipped_end(bam1_t *b) {
uint32_t *cigar = bam_get_cigar(b);
- int32_t end_pos, clipped = 0;
+ hts_pos_t end_pos, clipped = 0;
int32_t i;
end_pos = bam_endpos(b);
int i;
for (i = 0; i < b->core.l_qseq; i++) {
- if (qual[i] >= 15) score += qual[i];
+ if (qual[i] >= MD_MIN_QUALITY) score += qual[i];
}
return score;
the reference id, orientation and whether the current
read is leftmost of the pair. */
-static int make_pair_key(key_data_t *key, bam1_t *bam) {
- int32_t this_ref, this_coord, this_end;
- int32_t other_ref, other_coord, other_end;
- int32_t orientation, leftmost;
+static int make_pair_key_template(key_data_t *key, bam1_t *bam) {
+ hts_pos_t this_coord, other_coord, this_end, other_end;
+ int32_t this_ref, other_ref;
+ int8_t orientation, leftmost;
uint8_t *data;
char *cig;
this_end = unclipped_end(bam);
if ((data = bam_aux_get(bam, "MC"))) {
- cig = bam_aux2Z(data);
+ if (!(cig = bam_aux2Z(data))) {
+ fprintf(stderr, "[markdup] error: MC tag wrong type. Please use the MC tag provided by samtools fixmate.\n");
+ return 1;
+ }
+
other_end = unclipped_other_end(bam->core.mpos, cig);
other_coord = unclipped_other_start(bam->core.mpos, cig);
} else {
}
if (!leftmost)
- leftmost = 13;
+ leftmost = R_RI;
else
- leftmost = 11;
+ leftmost = R_LE;
key->single = 0;
key->this_ref = this_ref;
}
+static int make_pair_key_sequence(key_data_t *key, bam1_t *bam) {
+ hts_pos_t this_coord, this_end, other_coord, other_end, leftmost;
+ int32_t this_ref, other_ref;
+ int8_t orientation, left_read;
+ uint8_t *data;
+ char *cig;
+
+ this_ref = bam->core.tid + 1; // avoid a 0 being put into the hash
+ other_ref = bam->core.mtid + 1;
+
+ this_coord = unclipped_start(bam);
+ this_end = unclipped_end(bam);
+
+ if ((data = bam_aux_get(bam, "MC"))) {
+ if (!(cig = bam_aux2Z(data))) {
+ fprintf(stderr, "[markdup] error: MC tag wrong type. Please use the MC tag provided by samtools fixmate.\n");
+ return 1;
+ }
+
+ other_end = unclipped_other_end(bam->core.mpos, cig);
+ other_coord = unclipped_other_start(bam->core.mpos, cig);
+ } else {
+ fprintf(stderr, "[markdup] error: no MC tag. Please run samtools fixmate on file first.\n");
+ return 1;
+ }
+
+ // work out orientations
+ if (this_ref != other_ref) {
+ leftmost = this_ref - other_ref;
+ } else {
+ if (bam_is_rev(bam) == bam_is_mrev(bam)) {
+ if (!bam_is_rev(bam)) {
+ leftmost = this_coord - other_coord;
+ } else {
+ leftmost = this_end - other_end;
+ }
+ } else {
+ if (bam_is_rev(bam)) {
+ leftmost = this_end - other_coord;
+ } else {
+ leftmost = this_coord - other_end;
+ }
+ }
+ }
+
+ if (leftmost < 0) {
+ leftmost = 1;
+ } else if (leftmost > 0) {
+ leftmost = 0;
+ } else {
+ // tie breaks
+
+ if (bam->core.pos == bam->core.mpos) {
+ if (bam->core.flag & BAM_FREAD1) {
+ leftmost = 1;
+ } else {
+ leftmost = 0;
+ }
+ } else if (bam->core.pos < bam->core.mpos) {
+ leftmost = 1;
+ } else {
+ leftmost = 0;
+ }
+ }
+
+ // pair orientation
+ if (leftmost) {
+ if (bam_is_rev(bam) == bam_is_mrev(bam)) {
+
+ if (!bam_is_rev(bam)) {
+ orientation = O_FF;
+ } else {
+ orientation = O_RR;
+ }
+ } else {
+ if (!bam_is_rev(bam)) {
+ orientation = O_FR;
+ } else {
+ orientation = O_RF;
+ }
+ }
+ } else {
+ if (bam_is_rev(bam) == bam_is_mrev(bam)) {
+
+ if (!bam_is_rev(bam)) {
+ orientation = O_RR;
+ } else {
+ orientation = O_FF;
+ }
+ } else {
+ if (!bam_is_rev(bam)) {
+ orientation = O_RF;
+ } else {
+ orientation = O_FR;
+ }
+ }
+ }
+
+ if (!leftmost)
+ left_read = R_RI;
+ else
+ left_read = R_LE;
+
+ if (!bam_is_rev(bam)) {
+ this_coord = unclipped_start(bam);
+ } else {
+ this_coord = unclipped_end(bam);
+ }
+
+ if (!bam_is_mrev(bam)) {
+ other_coord = unclipped_other_start(bam->core.mpos, cig);
+ } else {
+ other_coord = unclipped_other_end(bam->core.mpos, cig);
+ }
+
+ key->single = 0;
+ key->this_ref = this_ref;
+ key->this_coord = this_coord;
+ key->other_ref = other_ref;
+ key->other_coord = other_coord;
+ key->leftmost = left_read;
+ key->orientation = orientation;
+
+ return 0;
+}
+
/* Create a signature hash of single read (or read with an unmatched pair).
Uses unclipped start (or end depending on orientation), reference id,
and orientation. */
static void make_single_key(key_data_t *key, bam1_t *bam) {
- int32_t this_ref, this_coord;
- int32_t orientation;
+ hts_pos_t this_coord;
+ int32_t this_ref;
+ int8_t orientation;
this_ref = bam->core.tid + 1; // avoid a 0 being put into the hash
key->orientation = orientation;
}
+
/* Add the duplicate name to a hash if it does not exist. */
-static int add_duplicate(khash_t(duplicates) *d_hash, bam1_t *dupe) {
+static int add_duplicate(khash_t(duplicates) *d_hash, bam1_t *dupe, char *orig_name, char type) {
khiter_t d;
int ret;
d = kh_get(duplicates, d_hash, bam_get_qname(dupe));
if (d == kh_end(d_hash)) {
- d = kh_put(duplicates, d_hash, strdup(bam_get_qname(dupe)), &ret);
+ char *name = strdup(bam_get_qname(dupe));
+ if (name) {
+ d = kh_put(duplicates, d_hash, name, &ret);
+ } else {
+ ret = -1;
+ }
+
+ if (ret >= 0) {
+ if (orig_name) {
+ if (ret == 0) {
+ // replace old name
+ free(kh_value(d_hash, d).name);
+ free(name);
+ }
+
+ kh_value(d_hash, d).name = strdup(orig_name);
- if (ret > 0) {
- kh_value(d_hash, d) = 1;
- } else if (ret == 0) {
- kh_value(d_hash, d)++;
+ if (kh_value(d_hash, d).name == NULL) {
+ fprintf(stderr, "[markdup] error: unable to allocate memory for duplicate original name.\n");
+ return 1;
+ }
+ } else {
+ kh_value(d_hash, d).name = NULL;
+ }
+
+ kh_value(d_hash, d).type = type;
} else {
fprintf(stderr, "[markdup] error: unable to store supplementary duplicates.\n");
+ free(name);
return 1;
}
}
}
+static inline int get_coordinate_positions(const char *qname, int *xpos, int *ypos) {
+ int sep = 0;
+ int pos = 0;
+
+ while (qname[pos]) {
+ if (qname[pos] == ':') {
+ sep++;
+
+ if (sep == 2) {
+ *xpos = pos + 1;
+ } else if (sep == 3) {
+ *ypos = pos + 1;
+ } else if (sep == 4) { // HiSeq style names
+ *xpos = *ypos;
+ *ypos = pos + 1;
+ } else if (sep == 5) { // Newer Illumina format
+ *xpos = pos + 1;
+ } else if (sep == 6) {
+ *ypos = pos + 1;
+ }
+ }
+
+ pos++;
+ }
+
+ return sep;
+}
+
+/* Using the coordinates from the Illumina read name, see whether the duplicated read is
+ close enough (set by max_dist) to the original to be counted as optical.*/
+
+static int optical_duplicate(bam1_t *ori, bam1_t *dup, long max_dist, long *warnings) {
+ int ret = 0, seps;
+ char *original, *duplicate;
+ int oxpos = 0, oypos = 0, dxpos = 0, dypos = 0;
+
+
+ original = bam_get_qname(ori);
+ duplicate = bam_get_qname(dup);
+
+ seps = get_coordinate_positions(original, &oxpos, &oypos);
+
+ if (!(seps == 3 || seps == 4 || seps == 6 || seps == 7)) {
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(stderr, "[markdup] warning: cannot decipher read name %s for optical duplicate marking.\n", original);
+ }
+
+ return ret;
+ }
+
+ seps = get_coordinate_positions(duplicate, &dxpos, &dypos);
+
+ if (!(seps == 3 || seps == 4 || seps == 6 || seps == 7)) {
+
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(stderr, "[markdup] warning: cannot decipher read name %s for optical duplicate marking.\n", duplicate);
+ }
+
+ return ret;
+ }
+
+ if (strncmp(original, duplicate, oxpos - 1) == 0) {
+ // the initial parts match, look at the numbers
+ long ox, oy, dx, dy, xdiff, ydiff;
+ char *end;
+
+ ox = strtol(original + oxpos, &end, 10);
+
+ if ((original + oxpos) == end) {
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(stderr, "[markdup] warning: can not decipher X coordinate in %s .\n", original);
+ }
+
+ return ret;
+ }
+
+ dx = strtol(duplicate + dxpos, &end, 10);
+
+ if ((duplicate + dxpos) == end) {
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(stderr, "[markdup] warning: can not decipher X coordinate in %s.\n", duplicate);
+ }
+
+ return ret;
+ }
+
+ if (ox > dx) {
+ xdiff = ox - dx;
+ } else {
+ xdiff = dx - ox;
+ }
+
+ if (xdiff <= max_dist) {
+ // still might be optical
+
+ oy = strtol(original + oypos, &end, 10);
+
+ if ((original + oypos) == end) {
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(stderr, "[markdup] warning: can not decipher Y coordinate in %s.\n", original);
+ }
+
+ return ret;
+ }
+
+ dy = strtol(duplicate + dypos, &end, 10);
+
+ if ((duplicate + dypos) == end) {
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(stderr, "[markdup] warning: can not decipher Y coordinate in %s.\n", duplicate);
+ }
+
+ return ret;
+ }
+
+ if (oy > dy) {
+ ydiff = oy - dy;
+ } else {
+ ydiff = dy - oy;
+ }
+
+ if (ydiff <= max_dist) ret = 1;
+ }
+ }
+
+ return ret;
+}
+
+
+static int mark_duplicates(md_param_t *param, khash_t(duplicates) *dup_hash, bam1_t *ori, bam1_t *dup,
+ long *optical, long *warn) {
+ char dup_type = 0;
+ long incoming_warnings = *warn;
+
+ dup->core.flag |= BAM_FDUP;
+
+ if (param->tag) {
+ if (bam_aux_append(dup, "do", 'Z', strlen(bam_get_qname(ori)) + 1, (uint8_t*)bam_get_qname(ori))) {
+ fprintf(stderr, "[markdup] error: unable to append 'do' tag.\n");
+ return -1;
+ }
+ }
+
+ if (param->opt_dist) { // mark optical duplicates
+ if (optical_duplicate(ori, dup, param->opt_dist, warn)) {
+ bam_aux_append(dup, "dt", 'Z', 3, (const uint8_t *)"SQ");
+ dup_type = 'O';
+ (*optical)++;
+ } else {
+ // not an optical duplicate
+ bam_aux_append(dup, "dt", 'Z', 3, (const uint8_t *)"LB");
+ }
+ }
+
+ if ((*warn == BMD_WARNING_MAX) && (incoming_warnings != *warn)) {
+ fprintf(stderr, "[markdup] warning: %ld decipher read name warnings. New warnings will not be reported.\n",
+ *warn);
+ }
+
+ if (param->supp) {
+ if (bam_aux_get(dup, "SA") || (dup->core.flag & BAM_FMUNMAP) || bam_aux_get(dup, "XA")) {
+ char *original = NULL;
+
+ if (param->tag) {
+ original = bam_get_qname(ori);
+ }
+
+ if (add_duplicate(dup_hash, dup, original, dup_type))
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+
+static inline int optical_retag(md_param_t *param, khash_t(duplicates) *dup_hash, bam1_t *b, int paired, long *optical_single, long *optical_pair) {
+ int ret = 0;
+ uint8_t *data;
+
+ // remove any existing dt tag
+ if ((data = bam_aux_get(b, "dt")) != NULL) {
+ bam_aux_del(b, data);
+ }
+
+ if (bam_aux_append(b, "dt", 'Z', 3, (const uint8_t *)"SQ")) {
+ fprintf(stderr, "[markdup] error: unable to append 'dt' tag.\n");
+ ret = -1;
+ }
+
+ if (paired) {
+ (*optical_pair)++;
+ } else {
+ (*optical_single)++;
+ }
+
+ if (param->supp) {
+ // Change the duplicate type
+
+ if (bam_aux_get(b, "SA") || (b->core.flag & BAM_FMUNMAP)
+ || bam_aux_get(b, "XA")) {
+ khiter_t d;
+
+ d = kh_get(duplicates, dup_hash, bam_get_qname(b));
+
+ if (d == kh_end(dup_hash)) {
+ // error, name should already be in dup hash
+ fprintf(stderr, "[markdup] error: duplicate name %s not found in hash.\n",
+ bam_get_qname(b));
+ ret = -1;
+ } else {
+ kh_value(dup_hash, d).type = 'O';
+ }
+ }
+ }
+
+ return ret;
+}
+
+
+
+/*
+ Where there is more than one duplicate go down the list and check for optical duplicates and change
+ do tags (where used) to point to original (non-duplicate) read.
+*/
+static int duplicate_chain_check(md_param_t *param, khash_t(duplicates) *dup_hash, read_queue_t *ori,
+ long *warn, long *optical_single, long *optical_pair) {
+ int ret = 0;
+ read_queue_t *current = ori->duplicate;
+ char *ori_name = bam_get_qname(ori->b);
+ int have_original = !(ori->b->core.flag & BAM_FDUP);
+ int ori_paired = (ori->b->core.flag & BAM_FPAIRED) && !(ori->b->core.flag & BAM_FMUNMAP);
+
+ while (current) {
+ int current_paired = (current->b->core.flag & BAM_FPAIRED) && !(current->b->core.flag & BAM_FMUNMAP);
+
+ if (param->tag && have_original) {
+ uint8_t *data;
+
+ // at this stage all duplicates should have a do tag
+ if ((data = bam_aux_get(current->b, "do")) != NULL) {
+ // see if we need to change the tag
+ char *old_name = bam_aux2Z(data);
+
+ if (old_name) {
+ if (strcmp(old_name, ori_name) != 0) {
+ bam_aux_del(current->b, data);
+
+ if (bam_aux_append(current->b, "do", 'Z', strlen(ori_name) + 1, (uint8_t*)ori_name)) {
+ fprintf(stderr, "[markdup] error: unable to append 'do' tag.\n");
+ ret = -1;
+ break;
+ }
+ }
+ } else {
+ fprintf(stderr, "[markdup] error: 'do' tag has wrong type for read %s.\n", bam_get_qname(current->b));
+ ret = -1;
+ break;
+ }
+ }
+ }
+
+ if (param->opt_dist) {
+ int is_cur_opt = 0, is_ori_opt = 0;
+ uint8_t *data;
+ char *dup_type;
+
+ if ((data = bam_aux_get(ori->b, "dt"))) {
+ if ((dup_type = bam_aux2Z(data))) {
+ if (strcmp(dup_type, "SQ") == 0) {
+ is_ori_opt = 1;
+ }
+ }
+ }
+
+ if ((data = bam_aux_get(current->b, "dt"))) {
+ if ((dup_type = bam_aux2Z(data))) {
+ if (strcmp(dup_type, "SQ") == 0) {
+ is_cur_opt = 1;
+ }
+ }
+ }
+
+ if (!(is_ori_opt && is_cur_opt)) {
+ // if both are already optical duplicates there is no need to check again, otherwise...
+
+ if (optical_duplicate(ori->b, current->b, param->opt_dist, warn)) {
+ // find out which one is the duplicate
+ int is_cur_dup = 0;
+
+ if (have_original) {
+ // compared against an original, this is a dup.
+ is_cur_dup = 1;
+ } else if (ori_paired != current_paired) {
+ if (!current_paired) {
+ // current is single vs pair, this is a dup.
+ is_cur_dup = 1;
+ }
+ } else {
+ // do it by scores
+ int64_t ori_score, curr_score;
+
+ if ((ori->b->core.flag & BAM_FQCFAIL) != (current->b->core.flag & BAM_FQCFAIL)) {
+ if (ori->b->core.flag & BAM_FQCFAIL) {
+ ori_score = 0;
+ curr_score = 1;
+ } else {
+ ori_score = 1;
+ curr_score = 0;
+ }
+ } else {
+ ori_score = calc_score(ori->b);
+ curr_score = calc_score(current->b);
+
+ if (current_paired) {
+ // they are pairs so add mate scores.
+ int64_t mate_tmp;
+
+ if ((mate_tmp = get_mate_score(ori->b)) == -1) {
+ fprintf(stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
+ ret = -1;
+ break;
+ } else {
+ ori_score += mate_tmp;
+ }
+
+ if ((mate_tmp = get_mate_score(current->b)) == -1) {
+ fprintf(stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
+ ret = -1;
+ break;
+ } else {
+ curr_score += mate_tmp;
+ }
+ }
+ }
+
+ if (ori_score == curr_score) {
+ if (strcmp(bam_get_qname(current->b), ori_name) < 0) {
+ curr_score++;
+ } else {
+ curr_score--;
+ }
+ }
+
+ if (ori_score > curr_score) {
+ is_cur_dup = 1;
+ }
+ }
+
+ if (is_cur_dup) {
+ // the current is the optical duplicate
+ if (!is_cur_opt) { // only change if not already an optical duplicate
+ if (optical_retag(param, dup_hash, current->b, current_paired, optical_single, optical_pair)) {
+ ret = -1;
+ break;
+ }
+ }
+ } else {
+ if (!is_ori_opt) {
+ if (optical_retag(param, dup_hash, ori->b, ori_paired, optical_single, optical_pair)) {
+ ret = -1;
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ current = current->duplicate;
+ }
+
+ return ret;
+}
+
+/*
+ Function to use when estimating library size.
+
+ This is based on an approximate formula for the coverage of a set
+ obtained after sampling it a given number of times with replacement.
+
+ x = number of items in the set (the number of unique fragments in the library)
+
+ c = number of unique items (unique read pairs observed)
+
+ n = number of items samples (total number of read pairs)
+
+ c and n are known; x is unknown.
+
+ As n -> infinity, the coverage (c/x) can be given as:
+
+ c / x = 1 - exp(-n / x) (see https://math.stackexchange.com/questions/32800)
+
+ This needs to be solved for x, so it is rearranged to put both terms on the
+ left side and estimate_library_size() finds a value of x which gives a
+ result of zero (or as close as it can get).
+ */
+static inline double coverage_equation(double x, double c, double n) {
+ return c / x - 1 + exp(-n / x);
+}
+
+
+/* estimate the library size, based on the Picard code in DuplicationMetrics.java*/
+static unsigned long estimate_library_size(unsigned long read_pairs, unsigned long duplicate_pairs) {
+ unsigned long estimated_size = 0;
+
+ read_pairs /= 2;
+ duplicate_pairs /= 2;
+
+ if ((read_pairs && duplicate_pairs) && (read_pairs > duplicate_pairs)) {
+ unsigned long unique_pairs = read_pairs - duplicate_pairs;
+ double m = 1;
+ double M = 100;
+ int i;
+
+ if (coverage_equation(m * (double)unique_pairs, (double)unique_pairs, (double)read_pairs) < 0) {
+ fprintf(stderr, "[markdup] warning: unable to calculate estimated library size.\n");
+ return estimated_size;
+ }
+
+ while (coverage_equation(M * (double)unique_pairs, (double)unique_pairs, (double)read_pairs) > 0) {
+ M *= 10;
+ }
+
+ for (i = 0; i < 40; i++) {
+ double r = (m + M) / 2;
+ double u = coverage_equation(r * (double)unique_pairs, (double)unique_pairs, (double)read_pairs);
+
+ if (u > 0) {
+ m = r;
+ } else if (u < 0) {
+ M = r;
+ } else {
+ break;
+ }
+ }
+
+ estimated_size = (unsigned long)(unique_pairs * (m + M) / 2);
+ } else {
+ fprintf(stderr, "[markdup] warning: unable to calculate estimated library size."
+ " Read pairs %ld should be greater than duplicate pairs %ld,"
+ " which should both be non zero.\n",
+ read_pairs, duplicate_pairs);
+ }
+
+ return estimated_size;
+}
+
+
/* Compare the reads near each other (coordinate sorted) and try to spot the duplicates.
Generally the highest quality scoring is chosen as the original and all others the duplicates.
The score is based on the sum of the quality values (<= 15) of the read and its mate (if any).
Marking the supplementary reads of a duplicate as also duplicates takes an extra file read/write
step. This is because the duplicate can occur before the primary read.*/
-static int bam_mark_duplicates(samFile *in, samFile *out, char *prefix, int remove_dups, int32_t max_length, int do_stats, int supp, int tag) {
- bam_hdr_t *header;
+static int bam_mark_duplicates(md_param_t *param) {
+ bam_hdr_t *header = NULL;
khiter_t k;
khash_t(reads) *pair_hash = kh_init(reads);
khash_t(reads) *single_hash = kh_init(reads);
klist_t(read_queue) *read_buffer = kl_init(read_queue);
kliter_t(read_queue) *rq;
khash_t(duplicates) *dup_hash = kh_init(duplicates);
- int32_t prev_tid, prev_coord;
+ int32_t prev_tid;
+ hts_pos_t prev_coord;
read_queue_t *in_read;
int ret;
- int reading, writing, excluded, duplicate, single, pair, single_dup, examined;
+ long reading, writing, excluded, duplicate, single, pair, single_dup, examined, optical, single_optical;
+ long np_duplicate, np_opt_duplicate;
+ long opt_warnings = 0;
tmp_file_t temp;
+ char *idx_fn = NULL;
+ int exclude = 0;
+
+ if (!pair_hash || !single_hash || !read_buffer || !dup_hash) {
+ fprintf(stderr, "[markdup] out of memory\n");
+ goto fail;
+ }
- if ((header = sam_hdr_read(in)) == NULL) {
+ if ((header = sam_hdr_read(param->in)) == NULL) {
fprintf(stderr, "[markdup] error reading header\n");
- return 1;
+ goto fail;
}
// accept unknown, unsorted or coordinate sort order, but error on queryname sorted.
// only really works on coordinate sorted files.
- if ((header->l_text > 3) && (strncmp(header->text, "@HD", 3) == 0)) {
- char *p, *q;
-
- p = strstr(header->text, "\tSO:queryname");
- q = strchr(header->text, '\n');
-
- // looking for SO:queryname within @HD only
- // (e.g. must ignore in a @CO comment line later in header)
- if ((p != 0) && (p < q)) {
- fprintf(stderr, "[markdup] error: queryname sorted, must be sorted by coordinate.\n");
- return 1;
- }
+ kstring_t str = KS_INITIALIZE;
+ if (!sam_hdr_find_tag_hd(header, "SO", &str) && str.s && !strcmp(str.s, "queryname")) {
+ fprintf(stderr, "[markdup] error: queryname sorted, must be sorted by coordinate.\n");
+ ks_free(&str);
+ goto fail;
+ }
+ ks_free(&str);
+
+ if (!param->no_pg && sam_hdr_add_pg(header, "samtools", "VN", samtools_version(),
+ param->arg_list ? "CL" : NULL,
+ param->arg_list ? param->arg_list : NULL,
+ NULL) != 0) {
+ fprintf(stderr, "[markdup] warning: unable to add @PG line to header.\n");
}
- if (sam_hdr_write(out, header) < 0) {
+ if (sam_hdr_write(param->out, header) < 0) {
fprintf(stderr, "[markdup] error writing header.\n");
- return 1;
+ goto fail;
+ }
+ if (param->write_index) {
+ if (!(idx_fn = auto_index(param->out, param->out_fn, header)))
+ goto fail;
}
// used for coordinate order checks
// get the buffer going
in_read = kl_pushp(read_queue, read_buffer);
+ if (!in_read) {
+ fprintf(stderr, "[markdup] out of memory\n");
+ goto fail;
+ }
// handling supplementary reads needs a temporary file
- if (supp) {
- if (tmp_file_open_write(&temp, prefix, 1)) {
- fprintf(stderr, "[markdup] error: unable to open tmp file %s.\n", prefix);
- return 1;
+ if (param->supp) {
+ if (tmp_file_open_write(&temp, param->prefix, 1)) {
+ fprintf(stderr, "[markdup] error: unable to open tmp file %s.\n", param->prefix);
+ goto fail;
}
}
if ((in_read->b = bam_init1()) == NULL) {
fprintf(stderr, "[markdup] error: unable to allocate memory for alignment.\n");
- return 1;
+ goto fail;
}
- reading = writing = excluded = single_dup = duplicate = examined = pair = single = 0;
+ reading = writing = excluded = single_dup = duplicate = examined = pair = single = optical = single_optical = 0;
+ np_duplicate = np_opt_duplicate = 0;
- while ((ret = sam_read1(in, header, in_read->b)) >= 0) {
+ while ((ret = sam_read1(param->in, header, in_read->b)) >= 0) {
// do some basic coordinate order checks
if (in_read->b->core.tid >= 0) { // -1 for unmapped reads
if (in_read->b->core.tid < prev_tid ||
((in_read->b->core.tid == prev_tid) && (in_read->b->core.pos < prev_coord))) {
- fprintf(stderr, "[markdup] error: bad coordinate order.\n");
- return 1;
+ fprintf(stderr, "[markdup] error: not in coordinate sorted order.\n");
+ goto fail;
}
}
reading++;
- // read must not be secondary, supplementary, unmapped or failed QC
- if (!(in_read->b->core.flag & (BAM_FSECONDARY | BAM_FSUPPLEMENTARY | BAM_FUNMAP | BAM_FQCFAIL))) {
- examined++;
+ if (param->clear && (in_read->b->core.flag & BAM_FDUP)) {
+ uint8_t *data;
+ in_read->b->core.flag ^= BAM_FDUP;
+
+ if ((data = bam_aux_get(in_read->b, "dt")) != NULL) {
+ bam_aux_del(in_read->b, data);
+ }
+
+ if ((data = bam_aux_get(in_read->b, "do")) != NULL) {
+ bam_aux_del(in_read->b, data);
+ }
+ }
+
+ if (param->include_fails) {
+ exclude |= (BAM_FSECONDARY | BAM_FSUPPLEMENTARY | BAM_FUNMAP);
+ } else {
+ exclude |= (BAM_FSECONDARY | BAM_FSUPPLEMENTARY | BAM_FUNMAP | BAM_FQCFAIL);
+ }
+
+ // read must not be secondary, supplementary, unmapped or (possibly) failed QC
+ if (!(in_read->b->core.flag & exclude)) {
+ examined++;
+ in_read->duplicate = NULL;
// look at the pairs first
if ((in_read->b->core.flag & BAM_FPAIRED) && !(in_read->b->core.flag & BAM_FMUNMAP)) {
key_data_t single_key;
in_hash_t *bp;
- if (make_pair_key(&pair_key, in_read->b)) {
- fprintf(stderr, "[markdup] error: unable to assign pair hash key.\n");
- return 1;
+ if (param->mode) {
+ if (make_pair_key_sequence(&pair_key, in_read->b)) {
+ fprintf(stderr, "[markdup] error: unable to assign pair hash key.\n");
+ goto fail;
+ }
+ } else {
+ if (make_pair_key_template(&pair_key, in_read->b)) {
+ fprintf(stderr, "[markdup] error: unable to assign pair hash key.\n");
+ goto fail;
+ }
}
make_single_key(&single_key, in_read->b);
if (ret > 0) { // new
// add to single duplicate hash
bp = &kh_val(single_hash, k);
- bp->p = in_read->b;
+ bp->p = in_read;
in_read->single_key = single_key;
} else if (ret == 0) { // exists
// look at singles only for duplication marking
bp = &kh_val(single_hash, k);
- if (!(bp->p->core.flag & BAM_FPAIRED) || (bp->p->core.flag & BAM_FMUNMAP)) {
- bam1_t *dup = bp->p;
+ if (!(bp->p->b->core.flag & BAM_FPAIRED) || (bp->p->b->core.flag & BAM_FMUNMAP)) {
+ // singleton will always be marked duplicate even if
+ // scores more than one read of the pair
+ bam1_t *dup = bp->p->b;
+
+ in_read->duplicate = bp->p;
+ bp->p = in_read;
- // singleton will always be marked duplicate even if
- // scores more than one read of the pair
+ if (mark_duplicates(param, dup_hash, bp->p->b, dup, &single_optical, &opt_warnings))
+ goto fail;
- bp->p = in_read->b;
- dup->core.flag |= BAM_FDUP;
single_dup++;
- if (tag) {
- if (bam_aux_append(dup, "do", 'Z', strlen(bam_get_qname(bp->p)) + 1, (uint8_t*)bam_get_qname(bp->p))) {
- fprintf(stderr, "[markdup] error: unable to append 'do' tag.\n");
- return 1;
- }
- }
+ if (duplicate_chain_check(param, dup_hash, bp->p, &opt_warnings, &single_optical, &optical))
+ goto fail;
- if (supp) {
- if (bam_aux_get(dup, "SA") || (dup->core.flag & BAM_FMUNMAP)) {
- if (add_duplicate(dup_hash, dup)) {
- return 1;
- }
- }
- }
}
} else {
fprintf(stderr, "[markdup] error: single hashing failure.\n");
- return 1;
+ goto fail;
}
// now do the pair
if (ret > 0) { // new
// add to the pair hash
bp = &kh_val(pair_hash, k);
- bp->p = in_read->b;
+ bp->p = in_read;
in_read->pair_key = pair_key;
} else if (ret == 0) {
int64_t old_score, new_score, tie_add = 0;
bam1_t *dup;
+ int check_chain = 0;
bp = &kh_val(pair_hash, k);
- if ((mate_tmp = get_mate_score(bp->p)) == -1) {
- fprintf(stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
- return 1;
+ if ((bp->p->b->core.flag & BAM_FQCFAIL) != (in_read->b->core.flag & BAM_FQCFAIL)) {
+ if (bp->p->b->core.flag & BAM_FQCFAIL) {
+ old_score = 0;
+ new_score = 1;
+ } else {
+ old_score = 1;
+ new_score = 0;
+ }
} else {
- old_score = calc_score(bp->p) + mate_tmp;
- }
+ if ((mate_tmp = get_mate_score(bp->p->b)) == -1) {
+ fprintf(stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
+ goto fail;
+ } else {
+ old_score = calc_score(bp->p->b) + mate_tmp;
+ }
- if ((mate_tmp = get_mate_score(in_read->b)) == -1) {
- fprintf(stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
- return 1;
- } else {
- new_score = calc_score(in_read->b) + mate_tmp;
+ if ((mate_tmp = get_mate_score(in_read->b)) == -1) {
+ fprintf(stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
+ goto fail;
+ } else {
+ new_score = calc_score(in_read->b) + mate_tmp;
+ }
}
// choose the highest score as the original
// and add it to the pair hash, mark the other as duplicate
if (new_score == old_score) {
- if (strcmp(bam_get_qname(in_read->b), bam_get_qname(bp->p)) < 0) {
+ if (strcmp(bam_get_qname(in_read->b), bam_get_qname(bp->p->b)) < 0) {
tie_add = 1;
} else {
tie_add = -1;
}
if (new_score + tie_add > old_score) { // swap reads
- dup = bp->p;
- bp->p = in_read->b;
+ dup = bp->p->b;
+ in_read->duplicate = bp->p;
+ bp->p = in_read;
} else {
+ if (bp->p->duplicate) {
+ in_read->duplicate = bp->p->duplicate;
+ check_chain = 1;
+ }
+
+ bp->p->duplicate = in_read;
dup = in_read->b;
}
- dup->core.flag |= BAM_FDUP;
-
- if (tag) {
- if (bam_aux_append(dup, "do", 'Z', strlen(bam_get_qname(bp->p)) + 1, (uint8_t*)bam_get_qname(bp->p))) {
- fprintf(stderr, "[markdup] error: unable to append 'do' tag.\n");
- return 1;
- }
+ if (mark_duplicates(param, dup_hash, bp->p->b, dup, &optical, &opt_warnings))
+ goto fail;
+ if (check_chain) {
+ if (duplicate_chain_check(param, dup_hash, bp->p->duplicate, &opt_warnings, &single_optical, &optical))
+ goto fail;
}
- if (supp) {
- if (bam_aux_get(dup, "SA") || (dup->core.flag & BAM_FMUNMAP)) {
- if (add_duplicate(dup_hash, dup)) {
- return 1;
- }
- }
- }
+ if (duplicate_chain_check(param, dup_hash, bp->p, &opt_warnings, &single_optical, &optical))
+ goto fail;
duplicate++;
} else {
fprintf(stderr, "[markdup] error: pair hashing failure.\n");
- return 1;
+ goto fail;
}
} else { // do the single (or effectively single) reads
int ret;
key_data_t single_key;
in_hash_t *bp;
+ int check_chain = 0;
make_single_key(&single_key, in_read->b);
if (ret > 0) { // new
bp = &kh_val(single_hash, k);
- bp->p = in_read->b;
+ bp->p = in_read;
in_read->single_key = single_key;
} else if (ret == 0) { // exists
bp = &kh_val(single_hash, k);
- if ((bp->p->core.flag & BAM_FPAIRED) && !(bp->p->core.flag & BAM_FMUNMAP)) {
+ if ((bp->p->b->core.flag & BAM_FPAIRED) && !(bp->p->b->core.flag & BAM_FMUNMAP)) {
// if matched against one of a pair just mark as duplicate
- if (tag) {
- if (bam_aux_append(in_read->b, "do", 'Z', strlen(bam_get_qname(bp->p)) + 1, (uint8_t*)bam_get_qname(bp->p))) {
- fprintf(stderr, "[markdup] error: unable to append 'do' tag.\n");
- return 1;
- }
+ if (bp->p->duplicate) {
+ in_read->duplicate = bp->p->duplicate;
+ check_chain = 1;
}
- if (supp) {
- if (bam_aux_get(in_read->b, "SA") || (in_read->b->core.flag & BAM_FMUNMAP)) {
- if (add_duplicate(dup_hash, in_read->b)) {
- return 1;
- }
- }
+ bp->p->duplicate = in_read;
+
+ if (mark_duplicates(param, dup_hash, bp->p->b, in_read->b, &single_optical, &opt_warnings))
+ goto fail;
+
+ if (check_chain) {
+ // check the new duplicate entry in the chain
+ if (duplicate_chain_check(param, dup_hash, bp->p->duplicate, &opt_warnings, &single_optical, &optical))
+ goto fail;
}
- in_read->b->core.flag |= BAM_FDUP;
+ // check against the new original
+ if (duplicate_chain_check(param, dup_hash, bp->p, &opt_warnings, &single_optical, &optical))
+ goto fail;
+
} else {
int64_t old_score, new_score;
bam1_t *dup;
- old_score = calc_score(bp->p);
+ old_score = calc_score(bp->p->b);
new_score = calc_score(in_read->b);
// choose the highest score as the original, add it
// to the single hash and mark the other as duplicate
if (new_score > old_score) { // swap reads
- dup = bp->p;
- bp->p = in_read->b;
+ dup = bp->p->b;
+ in_read->duplicate = bp->p;
+ bp->p = in_read;
} else {
+ if (bp->p->duplicate) {
+ in_read->duplicate = bp->p->duplicate;
+ check_chain = 1;
+ }
+
+ bp->p->duplicate = in_read;
dup = in_read->b;
}
- dup->core.flag |= BAM_FDUP;
+ if (mark_duplicates(param, dup_hash, bp->p->b, dup, &single_optical, &opt_warnings))
+ goto fail;
- if (tag) {
- if (bam_aux_append(dup, "do", 'Z', strlen(bam_get_qname(bp->p)) + 1, (uint8_t*)bam_get_qname(bp->p))) {
- fprintf(stderr, "[markdup] error: unable to append 'do' tag.\n");
- return 1;
- }
+
+ if (check_chain) {
+ if (duplicate_chain_check(param, dup_hash, bp->p->duplicate, &opt_warnings, &single_optical, &optical))
+ goto fail;
}
- if (supp) {
- if (bam_aux_get(dup, "SA") || (dup->core.flag & BAM_FMUNMAP)) {
- if (add_duplicate(dup_hash, dup)) {
- return 1;
- }
- }
+ if (duplicate_chain_check(param, dup_hash, bp->p, &opt_warnings, &single_optical, &optical))
+ goto fail;
+
+
}
- }
single_dup++;
} else {
fprintf(stderr, "[markdup] error: single hashing failure.\n");
- return 1;
+ goto fail;
}
}
} else {
/* keep a moving window of reads based on coordinates and max read length. Any unaligned reads
should just be written as they cannot be matched as duplicates. */
- if (in_read->pos + max_length > prev_coord && in_read->b->core.tid == prev_tid && (prev_tid != -1 || prev_coord != -1)) {
+ if (in_read->pos + param->max_length > prev_coord && in_read->b->core.tid == prev_tid && (prev_tid != -1 || prev_coord != -1)) {
break;
}
- if (!remove_dups || !(in_read->b->core.flag & BAM_FDUP)) {
- if (supp) {
+ if (!param->remove_dups || !(in_read->b->core.flag & BAM_FDUP)) {
+ if (param->supp) {
if (tmp_file_write(&temp, in_read->b)) {
fprintf(stderr, "[markdup] error: writing temp output failed.\n");
- return 1;
+ goto fail;
}
} else {
- if (sam_write1(out, header, in_read->b) < 0) {
+ if (sam_write1(param->out, header, in_read->b) < 0) {
fprintf(stderr, "[markdup] error: writing output failed.\n");
- return 1;
+ goto fail;
}
}
// set the next one up for reading
in_read = kl_pushp(read_queue, read_buffer);
+ if (!in_read) {
+ fprintf(stderr, "[markdup] out of memory\n");
+ goto fail;
+ }
if ((in_read->b = bam_init1()) == NULL) {
fprintf(stderr, "[markdup] error: unable to allocate memory for alignment.\n");
- return 1;
+ goto fail;
}
}
if (ret < -1) {
fprintf(stderr, "[markdup] error: truncated input file.\n");
- return 1;
+ goto fail;
}
// write out the end of the list
in_read = &kl_val(rq);
if (bam_get_qname(in_read->b)) { // last entry will be blank
- if (!remove_dups || !(in_read->b->core.flag & BAM_FDUP)) {
- if (supp) {
+ if (!param->remove_dups || !(in_read->b->core.flag & BAM_FDUP)) {
+ if (param->supp) {
if (tmp_file_write(&temp, in_read->b)) {
fprintf(stderr, "[markdup] error: writing temp output failed.\n");
- return 1;
+ goto fail;
}
} else {
- if (sam_write1(out, header, in_read->b) < 0) {
+ if (sam_write1(param->out, header, in_read->b) < 0) {
fprintf(stderr, "[markdup] error: writing output failed.\n");
- return 1;
+ goto fail;
}
}
rq = kl_begin(read_buffer);
}
- if (supp) {
+ if (param->supp) {
bam1_t *b;
if (tmp_file_end_write(&temp)) {
fprintf(stderr, "[markdup] error: unable to end tmp writing.\n");
- return 1;
+ goto fail;
}
// read data from temp file and mark duplicate supplementary alignments
- if (tmp_file_begin_read(&temp, NULL)) {
- return 1;
+ if (tmp_file_begin_read(&temp)) {
+ goto fail;
}
b = bam_init1();
while ((ret = tmp_file_read(&temp, b)) > 0) {
- if ((b->core.flag & BAM_FSUPPLEMENTARY) || (b->core.flag & BAM_FUNMAP)) {
+ if ((b->core.flag & BAM_FSUPPLEMENTARY) || (b->core.flag & BAM_FUNMAP) || (b->core.flag & BAM_FSECONDARY)) {
+
k = kh_get(duplicates, dup_hash, bam_get_qname(b));
if (k != kh_end(dup_hash)) {
+
b->core.flag |= BAM_FDUP;
+ np_duplicate++;
+
+ if (param->tag && kh_val(dup_hash, k).name) {
+ if (bam_aux_append(b, "do", 'Z', strlen(kh_val(dup_hash, k).name) + 1, (uint8_t*)kh_val(dup_hash, k).name)) {
+ fprintf(stderr, "[markdup] error: unable to append supplementary 'do' tag.\n");
+ goto fail;
+ }
+ }
+
+ if (param->opt_dist) {
+ if (kh_val(dup_hash, k).type) {
+ bam_aux_append(b, "dt", 'Z', 3, (const uint8_t *)"SQ");
+ np_opt_duplicate++;
+ } else {
+ bam_aux_append(b, "dt", 'Z', 3, (const uint8_t *)"LB");
+ }
+ }
}
}
- if (!remove_dups || !(b->core.flag & BAM_FDUP)) {
- if (sam_write1(out, header, b) < 0) {
+ if (!param->remove_dups || !(b->core.flag & BAM_FDUP)) {
+ if (sam_write1(param->out, header, b) < 0) {
fprintf(stderr, "[markdup] error: writing final output failed.\n");
- return 1;
+ goto fail;
}
}
}
if (ret == -1) {
fprintf(stderr, "[markdup] error: failed to read tmp file.\n");
- return 1;
+ goto fail;
}
for (k = kh_begin(dup_hash); k != kh_end(dup_hash); ++k) {
if (kh_exist(dup_hash, k)) {
+ free(kh_val(dup_hash, k).name);
free((char *)kh_key(dup_hash, k));
+ kh_key(dup_hash, k) = NULL;
}
}
- tmp_file_destroy(&temp, b, 0);
- kh_destroy(duplicates, dup_hash);
+ tmp_file_destroy(&temp);
bam_destroy1(b);
}
- if (do_stats) {
- fprintf(stderr, "READ %d WRITTEN %d \n"
- "EXCLUDED %d EXAMINED %d\n"
- "PAIRED %d SINGLE %d\n"
- "DULPICATE PAIR %d DUPLICATE SINGLE %d\n"
- "DUPLICATE TOTAL %d\n", reading, writing, excluded, examined, pair, single,
- duplicate, single_dup, single_dup + duplicate);
+ if (opt_warnings) {
+ fprintf(stderr, "[markdup] warning: number of failed attempts to get coordinates from read names = %ld\n",
+ opt_warnings);
+ }
+
+ if (param->do_stats) {
+ FILE *fp;
+ int file_open = 0;
+ unsigned long els;
+
+ if (param->stats_file) {
+ if (NULL == (fp = fopen(param->stats_file, "w"))) {
+ fprintf(stderr, "[markdup] warning: cannot write stats to %s.\n", param->stats_file);
+ fp = stderr;
+ } else {
+ file_open = 1;
+ }
+ } else {
+ fp = stderr;
+ }
+
+ els = estimate_library_size(pair, duplicate - optical);
+
+ fprintf(fp,
+ "COMMAND: %s\n"
+ "READ: %ld\n"
+ "WRITTEN: %ld\n"
+ "EXCLUDED: %ld\n"
+ "EXAMINED: %ld\n"
+ "PAIRED: %ld\n"
+ "SINGLE: %ld\n"
+ "DUPLICATE PAIR: %ld\n"
+ "DUPLICATE SINGLE: %ld\n"
+ "DUPLICATE PAIR OPTICAL: %ld\n"
+ "DUPLICATE SINGLE OPTICAL: %ld\n"
+ "DUPLICATE NON PRIMARY: %ld\n"
+ "DUPLICATE NON PRIMARY OPTICAL: %ld\n"
+ "DUPLICATE PRIMARY TOTAL: %ld\n"
+ "DUPLICATE TOTAL: %ld\n"
+ "ESTIMATED_LIBRARY_SIZE: %ld\n", param->arg_list, reading, writing, excluded, examined, pair, single,
+ duplicate, single_dup, optical, single_optical, np_duplicate, np_opt_duplicate,
+ single_dup + duplicate, single_dup + duplicate + np_duplicate, els);
+
+ if (file_open) {
+ fclose(fp);
+ }
+ }
+
+ if (param->write_index) {
+ if (sam_idx_save(param->out) < 0) {
+ print_error_errno("markdup", "writing index failed");
+ goto fail;
+ }
}
kh_destroy(reads, pair_hash);
kh_destroy(reads, single_hash);
kl_destroy(read_queue, read_buffer);
- bam_hdr_destroy(header);
+ kh_destroy(duplicates, dup_hash);
+ sam_hdr_destroy(header);
return 0;
+
+ fail:
+ for (rq = kl_begin(read_buffer); rq != kl_end(read_buffer); rq = kl_next(rq))
+ bam_destroy1(kl_val(rq).b);
+ kl_destroy(read_queue, read_buffer);
+
+ for (k = kh_begin(dup_hash); k != kh_end(dup_hash); ++k) {
+ if (kh_exist(dup_hash, k)) {
+ free((char *)kh_key(dup_hash, k));
+ }
+ }
+ kh_destroy(duplicates, dup_hash);
+
+ kh_destroy(reads, pair_hash);
+ kh_destroy(reads, single_hash);
+ sam_hdr_destroy(header);
+ return 1;
}
fprintf(stderr, "\n");
fprintf(stderr, "Usage: samtools markdup <input.bam> <output.bam>\n\n");
fprintf(stderr, "Option: \n");
- fprintf(stderr, " -r Remove duplicate reads\n");
- fprintf(stderr, " -l INT Max read length (default 300 bases)\n");
- fprintf(stderr, " -S Mark supplemenary alignments of duplicates as duplicates (slower).\n");
- fprintf(stderr, " -s Report stats.\n");
- fprintf(stderr, " -T PREFIX Write temporary files to PREFIX.samtools.nnnn.nnnn.tmp.\n");
- fprintf(stderr, " -t Mark primary duplicates with the name of the original in a \'do\' tag."
+ fprintf(stderr, " -r Remove duplicate reads\n");
+ fprintf(stderr, " -l INT Max read length (default 300 bases)\n");
+ fprintf(stderr, " -S Mark supplementary alignments of duplicates as duplicates (slower).\n");
+ fprintf(stderr, " -s Report stats.\n");
+ fprintf(stderr, " -f NAME Write stats to named file. Implies -s.\n");
+ fprintf(stderr, " -T PREFIX Write temporary files to PREFIX.samtools.nnnn.nnnn.tmp.\n");
+ fprintf(stderr, " -d INT Optical distance (if set, marks with dt tag)\n");
+ fprintf(stderr, " -c Clear previous duplicate settings and tags.\n");
+ fprintf(stderr, " -m --mode TYPE Duplicate decision method for paired reads.\n"
+ " TYPE = t measure positions based on template start/end (default).\n"
+ " s measure positions based on sequence start.\n");
+ fprintf(stderr, " --include-fails Include quality check failed reads.\n");
+ fprintf(stderr, " --no-PG Do not add a PG line\n");
+ fprintf(stderr, " -t Mark primary duplicates with the name of the original in a \'do\' tag."
" Mainly for information and debugging.\n");
- sam_global_opt_help(stderr, "-.O..@");
+ sam_global_opt_help(stderr, "-.O..@..");
fprintf(stderr, "\nThe input file must be coordinate sorted and must have gone"
" through fixmates with the mate scoring option on.\n");
int bam_markdup(int argc, char **argv) {
- int c, ret, remove_dups = 0, report_stats = 0, include_supplementary = 0, tag_dup = 0;
- int32_t max_length = 300;
- samFile *in = NULL, *out = NULL;
+ int c, ret;
char wmode[3] = {'w', 'b', 0};
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
htsThreadPool p = {NULL, 0};
kstring_t tmpprefix = {0, 0, NULL};
struct stat st;
unsigned int t;
+ md_param_t param = {NULL, NULL, NULL, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, NULL, NULL, NULL};
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 0, '@'),
+ {"include-fails", no_argument, NULL, 1001},
+ {"no-PG", no_argument, NULL, 1002},
+ {"mode", required_argument, NULL, 'm'},
{NULL, 0, NULL, 0}
};
- while ((c = getopt_long(argc, argv, "rsl:StT:O:@:", lopts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "rsl:StT:O:@:f:d:ncm:", lopts, NULL)) >= 0) {
switch (c) {
- case 'r': remove_dups = 1; break;
- case 'l': max_length = atoi(optarg); break;
- case 's': report_stats = 1; break;
+ case 'r': param.remove_dups = 1; break;
+ case 'l': param.max_length = atoi(optarg); break;
+ case 's': param.do_stats = 1; break;
case 'T': kputs(optarg, &tmpprefix); break;
- case 'S': include_supplementary = 1; break;
- case 't': tag_dup = 1; break;
+ case 'S': param.supp = 1; break;
+ case 't': param.tag = 1; break;
+ case 'f': param.stats_file = optarg; param.do_stats = 1; break;
+ case 'd': param.opt_dist = atoi(optarg); break;
+ case 'c': param.clear = 1; break;
+ case 'm':
+ if (strcmp(optarg, "t") == 0) {
+ param.mode = MD_MODE_TEMPLATE;
+ } else if (strcmp(optarg, "s") == 0) {
+ param.mode = MD_MODE_SEQUENCE;
+ } else {
+ fprintf(stderr, "[markdup] error: unknown mode '%s'.\n", optarg);
+ return markdup_usage();
+ }
+
+ break;
+ case 1001: param.include_fails = 1; break;
+ case 1002: param.no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': return markdup_usage();
if (optind + 2 > argc)
return markdup_usage();
- in = sam_open_format(argv[optind], "r", &ga.in);
+ if (param.opt_dist < 0) param.opt_dist = 0;
+ if (param.max_length < 0) param.max_length = 300;
+
+ param.in = sam_open_format(argv[optind], "r", &ga.in);
- if (!in) {
+ if (!param.in) {
print_error_errno("markdup", "failed to open \"%s\" for input", argv[optind]);
return 1;
}
sam_open_mode(wmode + 1, argv[optind + 1], NULL);
- out = sam_open_format(argv[optind + 1], wmode, &ga.out);
+ param.out = sam_open_format(argv[optind + 1], wmode, &ga.out);
- if (!out) {
+ if (!param.out) {
print_error_errno("markdup", "failed to open \"%s\" for output", argv[optind + 1]);
return 1;
}
return 1;
}
- hts_set_opt(in, HTS_OPT_THREAD_POOL, &p);
- hts_set_opt(out, HTS_OPT_THREAD_POOL, &p);
+ hts_set_opt(param.in, HTS_OPT_THREAD_POOL, &p);
+ hts_set_opt(param.out, HTS_OPT_THREAD_POOL, &p);
}
// actual stuff happens here
t = ((unsigned) time(NULL)) ^ ((unsigned) clock());
ksprintf(&tmpprefix, "samtools.%d.%u.tmp", (int) getpid(), t % 10000);
+ param.prefix = tmpprefix.s;
+
+ param.arg_list = stringify_argv(argc + 1, argv - 1);
+ param.write_index = ga.write_index;
+ param.out_fn = argv[optind + 1];
- ret = bam_mark_duplicates(in, out, tmpprefix.s, remove_dups, max_length, report_stats, include_supplementary, tag_dup);
+ ret = bam_mark_duplicates(¶m);
- sam_close(in);
+ sam_close(param.in);
- if (sam_close(out) < 0) {
+ if (sam_close(param.out) < 0) {
fprintf(stderr, "[markdup] error closing output file\n");
ret = 1;
}
if (p.pool) hts_tpool_destroy(p.pool);
+ free(param.arg_list);
free(tmpprefix.s);
sam_global_args_free(&ga);
/* bam_markdup.c -- Mark duplicates from a coord sorted file that has gone
through fixmates with the mate scoring option on.
- Copyright (C) 2017-18 Genome Research Ltd.
+ Copyright (C) 2017-2019 Genome Research Ltd.
Author: Andrew Whitwham <aw7@sanger.ac.uk>
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE
+
+Estimate library size derived from Picard DuplicationMetrics.java
+Copyright (c) 2009,2018 The Broad Institute. MIT license.
*/
#include <config.h>
#include <ctype.h>
#include <time.h>
#include <sys/stat.h>
+#include <math.h>
#include "htslib/thread_pool.h"
#include "htslib/sam.h"
#include "sam_opts.h"
#include "htslib/kstring.h"
#include "tmp_file.h"
+
+typedef struct {
+ samFile *in;
+ samFile *out;
+ char *prefix;
+ int remove_dups;
+ int32_t max_length;
+ int do_stats;
+ int supp;
+ int tag;
+ int opt_dist;
+ int no_pg;
+ int clear;
+ int mode;
+ int write_index;
+ int include_fails;
+ char *stats_file;
+ char *arg_list;
+ char *out_fn;
+} md_param_t;
+
typedef struct {
- int32_t single;
+ hts_pos_t this_coord;
+ hts_pos_t other_coord;
int32_t this_ref;
- int32_t this_coord;
int32_t other_ref;
- int32_t other_coord;
- int32_t leftmost;
- int32_t orientation;
+ int8_t single;
+ int8_t leftmost;
+ int8_t orientation;
} key_data_t;
+typedef struct read_queue_s {
+ key_data_t pair_key;
+ key_data_t single_key;
+ bam1_t *b;
+ struct read_queue_s *duplicate;
+ hts_pos_t pos;
+} read_queue_t;
+
typedef struct {
- bam1_t *p;
+ read_queue_t *p;
} in_hash_t;
typedef struct {
- bam1_t *b;
- int32_t pos;
- key_data_t pair_key;
- key_data_t single_key;
-} read_queue_t;
+ char *name;
+ char type;
+} dup_map_t;
khint_t hash;
if (key.single) {
- unsigned char sig[12];
+ unsigned char sig[13];
memcpy(sig + i, &key.this_ref, 4); i += 4;
- memcpy(sig + i, &key.this_coord, 4); i += 4;
- memcpy(sig + i, &key.orientation, 4); i += 4;
+ memcpy(sig + i, &key.this_coord, 8); i += 8;
+ memcpy(sig + i, &key.orientation, 1); i += 1;
hash = do_hash(sig, i);
} else {
- unsigned char sig[24];
+ unsigned char sig[26];
memcpy(sig + i, &key.this_ref, 4); i += 4;
- memcpy(sig + i, &key.this_coord, 4); i += 4;
+ memcpy(sig + i, &key.this_coord, 8); i += 8;
memcpy(sig + i, &key.other_ref, 4); i += 4;
- memcpy(sig + i, &key.other_coord, 4); i += 4;
- memcpy(sig + i, &key.leftmost, 4); i += 4;
- memcpy(sig + i, &key.orientation, 4); i += 4;
+ memcpy(sig + i, &key.other_coord, 8); i += 8;
+ memcpy(sig + i, &key.leftmost, 1); i += 1;
+ memcpy(sig + i, &key.orientation, 1); i += 1;
hash = do_hash(sig, i);
}
#define __free_queue_element(p)
+
+// Orientations (prime numbers to feed to hashing algorithm)
#define O_FF 2
#define O_RR 3
#define O_FR 5
#define O_RF 7
+// Left or rightmost
+#define R_LE 11
+#define R_RI 13
+
+#define BMD_WARNING_MAX 10
+
+#define MD_MIN_QUALITY 15
+
+// Duplicate finding mode
+#define MD_MODE_TEMPLATE 0
+#define MD_MODE_SEQUENCE 1
+
KHASH_INIT(reads, key_data_t, in_hash_t, 1, hash_key, key_equal) // read map hash
KLIST_INIT(read_queue, read_queue_t, __free_queue_element) // the reads buffer
-KHASH_MAP_INIT_STR(duplicates, int) // map of duplicates for supplementary dup id
+KHASH_MAP_INIT_STR(duplicates, dup_map_t) // map of duplicates for supplementary dup id
/* Calculate the mate's unclipped start based on position and cigar string from MC tag. */
-static int32_t unclipped_other_start(int32_t op, char *cigar) {
+static hts_pos_t unclipped_other_start(hts_pos_t op, char *cigar) {
char *c = cigar;
- int32_t clipped = 0;
+ int64_t clipped = 0;
while (*c && *c != '*') {
long num = 0;
/* Calculate the current read's start based on the stored cigar string. */
-static int32_t unclipped_start(bam1_t *b) {
+static hts_pos_t unclipped_start(bam1_t *b) {
uint32_t *cigar = bam_get_cigar(b);
- int32_t clipped = 0;
+ int64_t clipped = 0;
uint32_t i;
for (i = 0; i < b->core.n_cigar; i++) {
/* Calculate the mate's unclipped end based on start position and cigar string from MC tag.*/
-static int32_t unclipped_other_end(int32_t op, char *cigar) {
+static hts_pos_t unclipped_other_end(int64_t op, char *cigar) {
char *c = cigar;
- int32_t refpos = 0;
+ int64_t refpos = 0;
int skip = 1;
while (*c && *c != '*') {
/* Calculate the current read's end based on the stored cigar string. */
-static int32_t unclipped_end(bam1_t *b) {
+static hts_pos_t unclipped_end(bam1_t *b) {
uint32_t *cigar = bam_get_cigar(b);
- int32_t end_pos, clipped = 0;
+ hts_pos_t end_pos, clipped = 0;
int32_t i;
end_pos = bam_endpos(b);
int i;
for (i = 0; i < b->core.l_qseq; i++) {
- if (qual[i] >= 15) score += qual[i];
+ if (qual[i] >= MD_MIN_QUALITY) score += qual[i];
}
return score;
the reference id, orientation and whether the current
read is leftmost of the pair. */
-static int make_pair_key(key_data_t *key, bam1_t *bam) {
- int32_t this_ref, this_coord, this_end;
- int32_t other_ref, other_coord, other_end;
- int32_t orientation, leftmost;
+static int make_pair_key_template(key_data_t *key, bam1_t *bam) {
+ hts_pos_t this_coord, other_coord, this_end, other_end;
+ int32_t this_ref, other_ref;
+ int8_t orientation, leftmost;
uint8_t *data;
char *cig;
this_end = unclipped_end(bam);
if ((data = bam_aux_get(bam, "MC"))) {
- cig = bam_aux2Z(data);
+ if (!(cig = bam_aux2Z(data))) {
+ fprintf(samtools_stderr, "[markdup] error: MC tag wrong type. Please use the MC tag provided by samtools fixmate.\n");
+ return 1;
+ }
+
other_end = unclipped_other_end(bam->core.mpos, cig);
other_coord = unclipped_other_start(bam->core.mpos, cig);
} else {
}
if (!leftmost)
- leftmost = 13;
+ leftmost = R_RI;
else
- leftmost = 11;
+ leftmost = R_LE;
key->single = 0;
key->this_ref = this_ref;
}
+static int make_pair_key_sequence(key_data_t *key, bam1_t *bam) {
+ hts_pos_t this_coord, this_end, other_coord, other_end, leftmost;
+ int32_t this_ref, other_ref;
+ int8_t orientation, left_read;
+ uint8_t *data;
+ char *cig;
+
+ this_ref = bam->core.tid + 1; // avoid a 0 being put into the hash
+ other_ref = bam->core.mtid + 1;
+
+ this_coord = unclipped_start(bam);
+ this_end = unclipped_end(bam);
+
+ if ((data = bam_aux_get(bam, "MC"))) {
+ if (!(cig = bam_aux2Z(data))) {
+ fprintf(samtools_stderr, "[markdup] error: MC tag wrong type. Please use the MC tag provided by samtools fixmate.\n");
+ return 1;
+ }
+
+ other_end = unclipped_other_end(bam->core.mpos, cig);
+ other_coord = unclipped_other_start(bam->core.mpos, cig);
+ } else {
+ fprintf(samtools_stderr, "[markdup] error: no MC tag. Please run samtools fixmate on file first.\n");
+ return 1;
+ }
+
+ // work out orientations
+ if (this_ref != other_ref) {
+ leftmost = this_ref - other_ref;
+ } else {
+ if (bam_is_rev(bam) == bam_is_mrev(bam)) {
+ if (!bam_is_rev(bam)) {
+ leftmost = this_coord - other_coord;
+ } else {
+ leftmost = this_end - other_end;
+ }
+ } else {
+ if (bam_is_rev(bam)) {
+ leftmost = this_end - other_coord;
+ } else {
+ leftmost = this_coord - other_end;
+ }
+ }
+ }
+
+ if (leftmost < 0) {
+ leftmost = 1;
+ } else if (leftmost > 0) {
+ leftmost = 0;
+ } else {
+ // tie breaks
+
+ if (bam->core.pos == bam->core.mpos) {
+ if (bam->core.flag & BAM_FREAD1) {
+ leftmost = 1;
+ } else {
+ leftmost = 0;
+ }
+ } else if (bam->core.pos < bam->core.mpos) {
+ leftmost = 1;
+ } else {
+ leftmost = 0;
+ }
+ }
+
+ // pair orientation
+ if (leftmost) {
+ if (bam_is_rev(bam) == bam_is_mrev(bam)) {
+
+ if (!bam_is_rev(bam)) {
+ orientation = O_FF;
+ } else {
+ orientation = O_RR;
+ }
+ } else {
+ if (!bam_is_rev(bam)) {
+ orientation = O_FR;
+ } else {
+ orientation = O_RF;
+ }
+ }
+ } else {
+ if (bam_is_rev(bam) == bam_is_mrev(bam)) {
+
+ if (!bam_is_rev(bam)) {
+ orientation = O_RR;
+ } else {
+ orientation = O_FF;
+ }
+ } else {
+ if (!bam_is_rev(bam)) {
+ orientation = O_RF;
+ } else {
+ orientation = O_FR;
+ }
+ }
+ }
+
+ if (!leftmost)
+ left_read = R_RI;
+ else
+ left_read = R_LE;
+
+ if (!bam_is_rev(bam)) {
+ this_coord = unclipped_start(bam);
+ } else {
+ this_coord = unclipped_end(bam);
+ }
+
+ if (!bam_is_mrev(bam)) {
+ other_coord = unclipped_other_start(bam->core.mpos, cig);
+ } else {
+ other_coord = unclipped_other_end(bam->core.mpos, cig);
+ }
+
+ key->single = 0;
+ key->this_ref = this_ref;
+ key->this_coord = this_coord;
+ key->other_ref = other_ref;
+ key->other_coord = other_coord;
+ key->leftmost = left_read;
+ key->orientation = orientation;
+
+ return 0;
+}
+
/* Create a signature hash of single read (or read with an unmatched pair).
Uses unclipped start (or end depending on orientation), reference id,
and orientation. */
static void make_single_key(key_data_t *key, bam1_t *bam) {
- int32_t this_ref, this_coord;
- int32_t orientation;
+ hts_pos_t this_coord;
+ int32_t this_ref;
+ int8_t orientation;
this_ref = bam->core.tid + 1; // avoid a 0 being put into the hash
key->orientation = orientation;
}
+
/* Add the duplicate name to a hash if it does not exist. */
-static int add_duplicate(khash_t(duplicates) *d_hash, bam1_t *dupe) {
+static int add_duplicate(khash_t(duplicates) *d_hash, bam1_t *dupe, char *orig_name, char type) {
khiter_t d;
int ret;
d = kh_get(duplicates, d_hash, bam_get_qname(dupe));
if (d == kh_end(d_hash)) {
- d = kh_put(duplicates, d_hash, strdup(bam_get_qname(dupe)), &ret);
+ char *name = strdup(bam_get_qname(dupe));
+ if (name) {
+ d = kh_put(duplicates, d_hash, name, &ret);
+ } else {
+ ret = -1;
+ }
+
+ if (ret >= 0) {
+ if (orig_name) {
+ if (ret == 0) {
+ // replace old name
+ free(kh_value(d_hash, d).name);
+ free(name);
+ }
+
+ kh_value(d_hash, d).name = strdup(orig_name);
- if (ret > 0) {
- kh_value(d_hash, d) = 1;
- } else if (ret == 0) {
- kh_value(d_hash, d)++;
+ if (kh_value(d_hash, d).name == NULL) {
+ fprintf(samtools_stderr, "[markdup] error: unable to allocate memory for duplicate original name.\n");
+ return 1;
+ }
+ } else {
+ kh_value(d_hash, d).name = NULL;
+ }
+
+ kh_value(d_hash, d).type = type;
} else {
fprintf(samtools_stderr, "[markdup] error: unable to store supplementary duplicates.\n");
+ free(name);
return 1;
}
}
}
+static inline int get_coordinate_positions(const char *qname, int *xpos, int *ypos) {
+ int sep = 0;
+ int pos = 0;
+
+ while (qname[pos]) {
+ if (qname[pos] == ':') {
+ sep++;
+
+ if (sep == 2) {
+ *xpos = pos + 1;
+ } else if (sep == 3) {
+ *ypos = pos + 1;
+ } else if (sep == 4) { // HiSeq style names
+ *xpos = *ypos;
+ *ypos = pos + 1;
+ } else if (sep == 5) { // Newer Illumina format
+ *xpos = pos + 1;
+ } else if (sep == 6) {
+ *ypos = pos + 1;
+ }
+ }
+
+ pos++;
+ }
+
+ return sep;
+}
+
+/* Using the coordinates from the Illumina read name, see whether the duplicated read is
+ close enough (set by max_dist) to the original to be counted as optical.*/
+
+static int optical_duplicate(bam1_t *ori, bam1_t *dup, long max_dist, long *warnings) {
+ int ret = 0, seps;
+ char *original, *duplicate;
+ int oxpos = 0, oypos = 0, dxpos = 0, dypos = 0;
+
+
+ original = bam_get_qname(ori);
+ duplicate = bam_get_qname(dup);
+
+ seps = get_coordinate_positions(original, &oxpos, &oypos);
+
+ if (!(seps == 3 || seps == 4 || seps == 6 || seps == 7)) {
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(samtools_stderr, "[markdup] warning: cannot decipher read name %s for optical duplicate marking.\n", original);
+ }
+
+ return ret;
+ }
+
+ seps = get_coordinate_positions(duplicate, &dxpos, &dypos);
+
+ if (!(seps == 3 || seps == 4 || seps == 6 || seps == 7)) {
+
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(samtools_stderr, "[markdup] warning: cannot decipher read name %s for optical duplicate marking.\n", duplicate);
+ }
+
+ return ret;
+ }
+
+ if (strncmp(original, duplicate, oxpos - 1) == 0) {
+ // the initial parts match, look at the numbers
+ long ox, oy, dx, dy, xdiff, ydiff;
+ char *end;
+
+ ox = strtol(original + oxpos, &end, 10);
+
+ if ((original + oxpos) == end) {
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(samtools_stderr, "[markdup] warning: can not decipher X coordinate in %s .\n", original);
+ }
+
+ return ret;
+ }
+
+ dx = strtol(duplicate + dxpos, &end, 10);
+
+ if ((duplicate + dxpos) == end) {
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(samtools_stderr, "[markdup] warning: can not decipher X coordinate in %s.\n", duplicate);
+ }
+
+ return ret;
+ }
+
+ if (ox > dx) {
+ xdiff = ox - dx;
+ } else {
+ xdiff = dx - ox;
+ }
+
+ if (xdiff <= max_dist) {
+ // still might be optical
+
+ oy = strtol(original + oypos, &end, 10);
+
+ if ((original + oypos) == end) {
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(samtools_stderr, "[markdup] warning: can not decipher Y coordinate in %s.\n", original);
+ }
+
+ return ret;
+ }
+
+ dy = strtol(duplicate + dypos, &end, 10);
+
+ if ((duplicate + dypos) == end) {
+ (*warnings)++;
+
+ if (*warnings <= BMD_WARNING_MAX) {
+ fprintf(samtools_stderr, "[markdup] warning: can not decipher Y coordinate in %s.\n", duplicate);
+ }
+
+ return ret;
+ }
+
+ if (oy > dy) {
+ ydiff = oy - dy;
+ } else {
+ ydiff = dy - oy;
+ }
+
+ if (ydiff <= max_dist) ret = 1;
+ }
+ }
+
+ return ret;
+}
+
+
+static int mark_duplicates(md_param_t *param, khash_t(duplicates) *dup_hash, bam1_t *ori, bam1_t *dup,
+ long *optical, long *warn) {
+ char dup_type = 0;
+ long incoming_warnings = *warn;
+
+ dup->core.flag |= BAM_FDUP;
+
+ if (param->tag) {
+ if (bam_aux_append(dup, "do", 'Z', strlen(bam_get_qname(ori)) + 1, (uint8_t*)bam_get_qname(ori))) {
+ fprintf(samtools_stderr, "[markdup] error: unable to append 'do' tag.\n");
+ return -1;
+ }
+ }
+
+ if (param->opt_dist) { // mark optical duplicates
+ if (optical_duplicate(ori, dup, param->opt_dist, warn)) {
+ bam_aux_append(dup, "dt", 'Z', 3, (const uint8_t *)"SQ");
+ dup_type = 'O';
+ (*optical)++;
+ } else {
+ // not an optical duplicate
+ bam_aux_append(dup, "dt", 'Z', 3, (const uint8_t *)"LB");
+ }
+ }
+
+ if ((*warn == BMD_WARNING_MAX) && (incoming_warnings != *warn)) {
+ fprintf(samtools_stderr, "[markdup] warning: %ld decipher read name warnings. New warnings will not be reported.\n",
+ *warn);
+ }
+
+ if (param->supp) {
+ if (bam_aux_get(dup, "SA") || (dup->core.flag & BAM_FMUNMAP) || bam_aux_get(dup, "XA")) {
+ char *original = NULL;
+
+ if (param->tag) {
+ original = bam_get_qname(ori);
+ }
+
+ if (add_duplicate(dup_hash, dup, original, dup_type))
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+
+static inline int optical_retag(md_param_t *param, khash_t(duplicates) *dup_hash, bam1_t *b, int paired, long *optical_single, long *optical_pair) {
+ int ret = 0;
+ uint8_t *data;
+
+ // remove any existing dt tag
+ if ((data = bam_aux_get(b, "dt")) != NULL) {
+ bam_aux_del(b, data);
+ }
+
+ if (bam_aux_append(b, "dt", 'Z', 3, (const uint8_t *)"SQ")) {
+ fprintf(samtools_stderr, "[markdup] error: unable to append 'dt' tag.\n");
+ ret = -1;
+ }
+
+ if (paired) {
+ (*optical_pair)++;
+ } else {
+ (*optical_single)++;
+ }
+
+ if (param->supp) {
+ // Change the duplicate type
+
+ if (bam_aux_get(b, "SA") || (b->core.flag & BAM_FMUNMAP)
+ || bam_aux_get(b, "XA")) {
+ khiter_t d;
+
+ d = kh_get(duplicates, dup_hash, bam_get_qname(b));
+
+ if (d == kh_end(dup_hash)) {
+ // error, name should already be in dup hash
+ fprintf(samtools_stderr, "[markdup] error: duplicate name %s not found in hash.\n",
+ bam_get_qname(b));
+ ret = -1;
+ } else {
+ kh_value(dup_hash, d).type = 'O';
+ }
+ }
+ }
+
+ return ret;
+}
+
+
+
+/*
+ Where there is more than one duplicate go down the list and check for optical duplicates and change
+ do tags (where used) to point to original (non-duplicate) read.
+*/
+static int duplicate_chain_check(md_param_t *param, khash_t(duplicates) *dup_hash, read_queue_t *ori,
+ long *warn, long *optical_single, long *optical_pair) {
+ int ret = 0;
+ read_queue_t *current = ori->duplicate;
+ char *ori_name = bam_get_qname(ori->b);
+ int have_original = !(ori->b->core.flag & BAM_FDUP);
+ int ori_paired = (ori->b->core.flag & BAM_FPAIRED) && !(ori->b->core.flag & BAM_FMUNMAP);
+
+ while (current) {
+ int current_paired = (current->b->core.flag & BAM_FPAIRED) && !(current->b->core.flag & BAM_FMUNMAP);
+
+ if (param->tag && have_original) {
+ uint8_t *data;
+
+ // at this stage all duplicates should have a do tag
+ if ((data = bam_aux_get(current->b, "do")) != NULL) {
+ // see if we need to change the tag
+ char *old_name = bam_aux2Z(data);
+
+ if (old_name) {
+ if (strcmp(old_name, ori_name) != 0) {
+ bam_aux_del(current->b, data);
+
+ if (bam_aux_append(current->b, "do", 'Z', strlen(ori_name) + 1, (uint8_t*)ori_name)) {
+ fprintf(samtools_stderr, "[markdup] error: unable to append 'do' tag.\n");
+ ret = -1;
+ break;
+ }
+ }
+ } else {
+ fprintf(samtools_stderr, "[markdup] error: 'do' tag has wrong type for read %s.\n", bam_get_qname(current->b));
+ ret = -1;
+ break;
+ }
+ }
+ }
+
+ if (param->opt_dist) {
+ int is_cur_opt = 0, is_ori_opt = 0;
+ uint8_t *data;
+ char *dup_type;
+
+ if ((data = bam_aux_get(ori->b, "dt"))) {
+ if ((dup_type = bam_aux2Z(data))) {
+ if (strcmp(dup_type, "SQ") == 0) {
+ is_ori_opt = 1;
+ }
+ }
+ }
+
+ if ((data = bam_aux_get(current->b, "dt"))) {
+ if ((dup_type = bam_aux2Z(data))) {
+ if (strcmp(dup_type, "SQ") == 0) {
+ is_cur_opt = 1;
+ }
+ }
+ }
+
+ if (!(is_ori_opt && is_cur_opt)) {
+ // if both are already optical duplicates there is no need to check again, otherwise...
+
+ if (optical_duplicate(ori->b, current->b, param->opt_dist, warn)) {
+ // find out which one is the duplicate
+ int is_cur_dup = 0;
+
+ if (have_original) {
+ // compared against an original, this is a dup.
+ is_cur_dup = 1;
+ } else if (ori_paired != current_paired) {
+ if (!current_paired) {
+ // current is single vs pair, this is a dup.
+ is_cur_dup = 1;
+ }
+ } else {
+ // do it by scores
+ int64_t ori_score, curr_score;
+
+ if ((ori->b->core.flag & BAM_FQCFAIL) != (current->b->core.flag & BAM_FQCFAIL)) {
+ if (ori->b->core.flag & BAM_FQCFAIL) {
+ ori_score = 0;
+ curr_score = 1;
+ } else {
+ ori_score = 1;
+ curr_score = 0;
+ }
+ } else {
+ ori_score = calc_score(ori->b);
+ curr_score = calc_score(current->b);
+
+ if (current_paired) {
+ // they are pairs so add mate scores.
+ int64_t mate_tmp;
+
+ if ((mate_tmp = get_mate_score(ori->b)) == -1) {
+ fprintf(samtools_stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
+ ret = -1;
+ break;
+ } else {
+ ori_score += mate_tmp;
+ }
+
+ if ((mate_tmp = get_mate_score(current->b)) == -1) {
+ fprintf(samtools_stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
+ ret = -1;
+ break;
+ } else {
+ curr_score += mate_tmp;
+ }
+ }
+ }
+
+ if (ori_score == curr_score) {
+ if (strcmp(bam_get_qname(current->b), ori_name) < 0) {
+ curr_score++;
+ } else {
+ curr_score--;
+ }
+ }
+
+ if (ori_score > curr_score) {
+ is_cur_dup = 1;
+ }
+ }
+
+ if (is_cur_dup) {
+ // the current is the optical duplicate
+ if (!is_cur_opt) { // only change if not already an optical duplicate
+ if (optical_retag(param, dup_hash, current->b, current_paired, optical_single, optical_pair)) {
+ ret = -1;
+ break;
+ }
+ }
+ } else {
+ if (!is_ori_opt) {
+ if (optical_retag(param, dup_hash, ori->b, ori_paired, optical_single, optical_pair)) {
+ ret = -1;
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ current = current->duplicate;
+ }
+
+ return ret;
+}
+
+/*
+ Function to use when estimating library size.
+
+ This is based on an approximate formula for the coverage of a set
+ obtained after sampling it a given number of times with replacement.
+
+ x = number of items in the set (the number of unique fragments in the library)
+
+ c = number of unique items (unique read pairs observed)
+
+ n = number of items samples (total number of read pairs)
+
+ c and n are known; x is unknown.
+
+ As n -> infinity, the coverage (c/x) can be given as:
+
+ c / x = 1 - exp(-n / x) (see https://math.stackexchange.com/questions/32800)
+
+ This needs to be solved for x, so it is rearranged to put both terms on the
+ left side and estimate_library_size() finds a value of x which gives a
+ result of zero (or as close as it can get).
+ */
+static inline double coverage_equation(double x, double c, double n) {
+ return c / x - 1 + exp(-n / x);
+}
+
+
+/* estimate the library size, based on the Picard code in DuplicationMetrics.java*/
+static unsigned long estimate_library_size(unsigned long read_pairs, unsigned long duplicate_pairs) {
+ unsigned long estimated_size = 0;
+
+ read_pairs /= 2;
+ duplicate_pairs /= 2;
+
+ if ((read_pairs && duplicate_pairs) && (read_pairs > duplicate_pairs)) {
+ unsigned long unique_pairs = read_pairs - duplicate_pairs;
+ double m = 1;
+ double M = 100;
+ int i;
+
+ if (coverage_equation(m * (double)unique_pairs, (double)unique_pairs, (double)read_pairs) < 0) {
+ fprintf(samtools_stderr, "[markdup] warning: unable to calculate estimated library size.\n");
+ return estimated_size;
+ }
+
+ while (coverage_equation(M * (double)unique_pairs, (double)unique_pairs, (double)read_pairs) > 0) {
+ M *= 10;
+ }
+
+ for (i = 0; i < 40; i++) {
+ double r = (m + M) / 2;
+ double u = coverage_equation(r * (double)unique_pairs, (double)unique_pairs, (double)read_pairs);
+
+ if (u > 0) {
+ m = r;
+ } else if (u < 0) {
+ M = r;
+ } else {
+ break;
+ }
+ }
+
+ estimated_size = (unsigned long)(unique_pairs * (m + M) / 2);
+ } else {
+ fprintf(samtools_stderr, "[markdup] warning: unable to calculate estimated library size."
+ " Read pairs %ld should be greater than duplicate pairs %ld,"
+ " which should both be non zero.\n",
+ read_pairs, duplicate_pairs);
+ }
+
+ return estimated_size;
+}
+
+
/* Compare the reads near each other (coordinate sorted) and try to spot the duplicates.
Generally the highest quality scoring is chosen as the original and all others the duplicates.
The score is based on the sum of the quality values (<= 15) of the read and its mate (if any).
Marking the supplementary reads of a duplicate as also duplicates takes an extra file read/write
step. This is because the duplicate can occur before the primary read.*/
-static int bam_mark_duplicates(samFile *in, samFile *out, char *prefix, int remove_dups, int32_t max_length, int do_stats, int supp, int tag) {
- bam_hdr_t *header;
+static int bam_mark_duplicates(md_param_t *param) {
+ bam_hdr_t *header = NULL;
khiter_t k;
khash_t(reads) *pair_hash = kh_init(reads);
khash_t(reads) *single_hash = kh_init(reads);
klist_t(read_queue) *read_buffer = kl_init(read_queue);
kliter_t(read_queue) *rq;
khash_t(duplicates) *dup_hash = kh_init(duplicates);
- int32_t prev_tid, prev_coord;
+ int32_t prev_tid;
+ hts_pos_t prev_coord;
read_queue_t *in_read;
int ret;
- int reading, writing, excluded, duplicate, single, pair, single_dup, examined;
+ long reading, writing, excluded, duplicate, single, pair, single_dup, examined, optical, single_optical;
+ long np_duplicate, np_opt_duplicate;
+ long opt_warnings = 0;
tmp_file_t temp;
+ char *idx_fn = NULL;
+ int exclude = 0;
+
+ if (!pair_hash || !single_hash || !read_buffer || !dup_hash) {
+ fprintf(samtools_stderr, "[markdup] out of memory\n");
+ goto fail;
+ }
- if ((header = sam_hdr_read(in)) == NULL) {
+ if ((header = sam_hdr_read(param->in)) == NULL) {
fprintf(samtools_stderr, "[markdup] error reading header\n");
- return 1;
+ goto fail;
}
// accept unknown, unsorted or coordinate sort order, but error on queryname sorted.
// only really works on coordinate sorted files.
- if ((header->l_text > 3) && (strncmp(header->text, "@HD", 3) == 0)) {
- char *p, *q;
-
- p = strstr(header->text, "\tSO:queryname");
- q = strchr(header->text, '\n');
-
- // looking for SO:queryname within @HD only
- // (e.g. must ignore in a @CO comment line later in header)
- if ((p != 0) && (p < q)) {
- fprintf(samtools_stderr, "[markdup] error: queryname sorted, must be sorted by coordinate.\n");
- return 1;
- }
+ kstring_t str = KS_INITIALIZE;
+ if (!sam_hdr_find_tag_hd(header, "SO", &str) && str.s && !strcmp(str.s, "queryname")) {
+ fprintf(samtools_stderr, "[markdup] error: queryname sorted, must be sorted by coordinate.\n");
+ ks_free(&str);
+ goto fail;
+ }
+ ks_free(&str);
+
+ if (!param->no_pg && sam_hdr_add_pg(header, "samtools", "VN", samtools_version(),
+ param->arg_list ? "CL" : NULL,
+ param->arg_list ? param->arg_list : NULL,
+ NULL) != 0) {
+ fprintf(samtools_stderr, "[markdup] warning: unable to add @PG line to header.\n");
}
- if (sam_hdr_write(out, header) < 0) {
+ if (sam_hdr_write(param->out, header) < 0) {
fprintf(samtools_stderr, "[markdup] error writing header.\n");
- return 1;
+ goto fail;
+ }
+ if (param->write_index) {
+ if (!(idx_fn = auto_index(param->out, param->out_fn, header)))
+ goto fail;
}
// used for coordinate order checks
// get the buffer going
in_read = kl_pushp(read_queue, read_buffer);
+ if (!in_read) {
+ fprintf(samtools_stderr, "[markdup] out of memory\n");
+ goto fail;
+ }
// handling supplementary reads needs a temporary file
- if (supp) {
- if (tmp_file_open_write(&temp, prefix, 1)) {
- fprintf(samtools_stderr, "[markdup] error: unable to open tmp file %s.\n", prefix);
- return 1;
+ if (param->supp) {
+ if (tmp_file_open_write(&temp, param->prefix, 1)) {
+ fprintf(samtools_stderr, "[markdup] error: unable to open tmp file %s.\n", param->prefix);
+ goto fail;
}
}
if ((in_read->b = bam_init1()) == NULL) {
fprintf(samtools_stderr, "[markdup] error: unable to allocate memory for alignment.\n");
- return 1;
+ goto fail;
}
- reading = writing = excluded = single_dup = duplicate = examined = pair = single = 0;
+ reading = writing = excluded = single_dup = duplicate = examined = pair = single = optical = single_optical = 0;
+ np_duplicate = np_opt_duplicate = 0;
- while ((ret = sam_read1(in, header, in_read->b)) >= 0) {
+ while ((ret = sam_read1(param->in, header, in_read->b)) >= 0) {
// do some basic coordinate order checks
if (in_read->b->core.tid >= 0) { // -1 for unmapped reads
if (in_read->b->core.tid < prev_tid ||
((in_read->b->core.tid == prev_tid) && (in_read->b->core.pos < prev_coord))) {
- fprintf(samtools_stderr, "[markdup] error: bad coordinate order.\n");
- return 1;
+ fprintf(samtools_stderr, "[markdup] error: not in coordinate sorted order.\n");
+ goto fail;
}
}
reading++;
- // read must not be secondary, supplementary, unmapped or failed QC
- if (!(in_read->b->core.flag & (BAM_FSECONDARY | BAM_FSUPPLEMENTARY | BAM_FUNMAP | BAM_FQCFAIL))) {
- examined++;
+ if (param->clear && (in_read->b->core.flag & BAM_FDUP)) {
+ uint8_t *data;
+ in_read->b->core.flag ^= BAM_FDUP;
+
+ if ((data = bam_aux_get(in_read->b, "dt")) != NULL) {
+ bam_aux_del(in_read->b, data);
+ }
+
+ if ((data = bam_aux_get(in_read->b, "do")) != NULL) {
+ bam_aux_del(in_read->b, data);
+ }
+ }
+
+ if (param->include_fails) {
+ exclude |= (BAM_FSECONDARY | BAM_FSUPPLEMENTARY | BAM_FUNMAP);
+ } else {
+ exclude |= (BAM_FSECONDARY | BAM_FSUPPLEMENTARY | BAM_FUNMAP | BAM_FQCFAIL);
+ }
+
+ // read must not be secondary, supplementary, unmapped or (possibly) failed QC
+ if (!(in_read->b->core.flag & exclude)) {
+ examined++;
+ in_read->duplicate = NULL;
// look at the pairs first
if ((in_read->b->core.flag & BAM_FPAIRED) && !(in_read->b->core.flag & BAM_FMUNMAP)) {
key_data_t single_key;
in_hash_t *bp;
- if (make_pair_key(&pair_key, in_read->b)) {
- fprintf(samtools_stderr, "[markdup] error: unable to assign pair hash key.\n");
- return 1;
+ if (param->mode) {
+ if (make_pair_key_sequence(&pair_key, in_read->b)) {
+ fprintf(samtools_stderr, "[markdup] error: unable to assign pair hash key.\n");
+ goto fail;
+ }
+ } else {
+ if (make_pair_key_template(&pair_key, in_read->b)) {
+ fprintf(samtools_stderr, "[markdup] error: unable to assign pair hash key.\n");
+ goto fail;
+ }
}
make_single_key(&single_key, in_read->b);
if (ret > 0) { // new
// add to single duplicate hash
bp = &kh_val(single_hash, k);
- bp->p = in_read->b;
+ bp->p = in_read;
in_read->single_key = single_key;
} else if (ret == 0) { // exists
// look at singles only for duplication marking
bp = &kh_val(single_hash, k);
- if (!(bp->p->core.flag & BAM_FPAIRED) || (bp->p->core.flag & BAM_FMUNMAP)) {
- bam1_t *dup = bp->p;
+ if (!(bp->p->b->core.flag & BAM_FPAIRED) || (bp->p->b->core.flag & BAM_FMUNMAP)) {
+ // singleton will always be marked duplicate even if
+ // scores more than one read of the pair
+ bam1_t *dup = bp->p->b;
+
+ in_read->duplicate = bp->p;
+ bp->p = in_read;
- // singleton will always be marked duplicate even if
- // scores more than one read of the pair
+ if (mark_duplicates(param, dup_hash, bp->p->b, dup, &single_optical, &opt_warnings))
+ goto fail;
- bp->p = in_read->b;
- dup->core.flag |= BAM_FDUP;
single_dup++;
- if (tag) {
- if (bam_aux_append(dup, "do", 'Z', strlen(bam_get_qname(bp->p)) + 1, (uint8_t*)bam_get_qname(bp->p))) {
- fprintf(samtools_stderr, "[markdup] error: unable to append 'do' tag.\n");
- return 1;
- }
- }
+ if (duplicate_chain_check(param, dup_hash, bp->p, &opt_warnings, &single_optical, &optical))
+ goto fail;
- if (supp) {
- if (bam_aux_get(dup, "SA") || (dup->core.flag & BAM_FMUNMAP)) {
- if (add_duplicate(dup_hash, dup)) {
- return 1;
- }
- }
- }
}
} else {
fprintf(samtools_stderr, "[markdup] error: single hashing failure.\n");
- return 1;
+ goto fail;
}
// now do the pair
if (ret > 0) { // new
// add to the pair hash
bp = &kh_val(pair_hash, k);
- bp->p = in_read->b;
+ bp->p = in_read;
in_read->pair_key = pair_key;
} else if (ret == 0) {
int64_t old_score, new_score, tie_add = 0;
bam1_t *dup;
+ int check_chain = 0;
bp = &kh_val(pair_hash, k);
- if ((mate_tmp = get_mate_score(bp->p)) == -1) {
- fprintf(samtools_stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
- return 1;
+ if ((bp->p->b->core.flag & BAM_FQCFAIL) != (in_read->b->core.flag & BAM_FQCFAIL)) {
+ if (bp->p->b->core.flag & BAM_FQCFAIL) {
+ old_score = 0;
+ new_score = 1;
+ } else {
+ old_score = 1;
+ new_score = 0;
+ }
} else {
- old_score = calc_score(bp->p) + mate_tmp;
- }
+ if ((mate_tmp = get_mate_score(bp->p->b)) == -1) {
+ fprintf(samtools_stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
+ goto fail;
+ } else {
+ old_score = calc_score(bp->p->b) + mate_tmp;
+ }
- if ((mate_tmp = get_mate_score(in_read->b)) == -1) {
- fprintf(samtools_stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
- return 1;
- } else {
- new_score = calc_score(in_read->b) + mate_tmp;
+ if ((mate_tmp = get_mate_score(in_read->b)) == -1) {
+ fprintf(samtools_stderr, "[markdup] error: no ms score tag. Please run samtools fixmate on file first.\n");
+ goto fail;
+ } else {
+ new_score = calc_score(in_read->b) + mate_tmp;
+ }
}
// choose the highest score as the original
// and add it to the pair hash, mark the other as duplicate
if (new_score == old_score) {
- if (strcmp(bam_get_qname(in_read->b), bam_get_qname(bp->p)) < 0) {
+ if (strcmp(bam_get_qname(in_read->b), bam_get_qname(bp->p->b)) < 0) {
tie_add = 1;
} else {
tie_add = -1;
}
if (new_score + tie_add > old_score) { // swap reads
- dup = bp->p;
- bp->p = in_read->b;
+ dup = bp->p->b;
+ in_read->duplicate = bp->p;
+ bp->p = in_read;
} else {
+ if (bp->p->duplicate) {
+ in_read->duplicate = bp->p->duplicate;
+ check_chain = 1;
+ }
+
+ bp->p->duplicate = in_read;
dup = in_read->b;
}
- dup->core.flag |= BAM_FDUP;
-
- if (tag) {
- if (bam_aux_append(dup, "do", 'Z', strlen(bam_get_qname(bp->p)) + 1, (uint8_t*)bam_get_qname(bp->p))) {
- fprintf(samtools_stderr, "[markdup] error: unable to append 'do' tag.\n");
- return 1;
- }
+ if (mark_duplicates(param, dup_hash, bp->p->b, dup, &optical, &opt_warnings))
+ goto fail;
+ if (check_chain) {
+ if (duplicate_chain_check(param, dup_hash, bp->p->duplicate, &opt_warnings, &single_optical, &optical))
+ goto fail;
}
- if (supp) {
- if (bam_aux_get(dup, "SA") || (dup->core.flag & BAM_FMUNMAP)) {
- if (add_duplicate(dup_hash, dup)) {
- return 1;
- }
- }
- }
+ if (duplicate_chain_check(param, dup_hash, bp->p, &opt_warnings, &single_optical, &optical))
+ goto fail;
duplicate++;
} else {
fprintf(samtools_stderr, "[markdup] error: pair hashing failure.\n");
- return 1;
+ goto fail;
}
} else { // do the single (or effectively single) reads
int ret;
key_data_t single_key;
in_hash_t *bp;
+ int check_chain = 0;
make_single_key(&single_key, in_read->b);
if (ret > 0) { // new
bp = &kh_val(single_hash, k);
- bp->p = in_read->b;
+ bp->p = in_read;
in_read->single_key = single_key;
} else if (ret == 0) { // exists
bp = &kh_val(single_hash, k);
- if ((bp->p->core.flag & BAM_FPAIRED) && !(bp->p->core.flag & BAM_FMUNMAP)) {
+ if ((bp->p->b->core.flag & BAM_FPAIRED) && !(bp->p->b->core.flag & BAM_FMUNMAP)) {
// if matched against one of a pair just mark as duplicate
- if (tag) {
- if (bam_aux_append(in_read->b, "do", 'Z', strlen(bam_get_qname(bp->p)) + 1, (uint8_t*)bam_get_qname(bp->p))) {
- fprintf(samtools_stderr, "[markdup] error: unable to append 'do' tag.\n");
- return 1;
- }
+ if (bp->p->duplicate) {
+ in_read->duplicate = bp->p->duplicate;
+ check_chain = 1;
}
- if (supp) {
- if (bam_aux_get(in_read->b, "SA") || (in_read->b->core.flag & BAM_FMUNMAP)) {
- if (add_duplicate(dup_hash, in_read->b)) {
- return 1;
- }
- }
+ bp->p->duplicate = in_read;
+
+ if (mark_duplicates(param, dup_hash, bp->p->b, in_read->b, &single_optical, &opt_warnings))
+ goto fail;
+
+ if (check_chain) {
+ // check the new duplicate entry in the chain
+ if (duplicate_chain_check(param, dup_hash, bp->p->duplicate, &opt_warnings, &single_optical, &optical))
+ goto fail;
}
- in_read->b->core.flag |= BAM_FDUP;
+ // check against the new original
+ if (duplicate_chain_check(param, dup_hash, bp->p, &opt_warnings, &single_optical, &optical))
+ goto fail;
+
} else {
int64_t old_score, new_score;
bam1_t *dup;
- old_score = calc_score(bp->p);
+ old_score = calc_score(bp->p->b);
new_score = calc_score(in_read->b);
// choose the highest score as the original, add it
// to the single hash and mark the other as duplicate
if (new_score > old_score) { // swap reads
- dup = bp->p;
- bp->p = in_read->b;
+ dup = bp->p->b;
+ in_read->duplicate = bp->p;
+ bp->p = in_read;
} else {
+ if (bp->p->duplicate) {
+ in_read->duplicate = bp->p->duplicate;
+ check_chain = 1;
+ }
+
+ bp->p->duplicate = in_read;
dup = in_read->b;
}
- dup->core.flag |= BAM_FDUP;
+ if (mark_duplicates(param, dup_hash, bp->p->b, dup, &single_optical, &opt_warnings))
+ goto fail;
- if (tag) {
- if (bam_aux_append(dup, "do", 'Z', strlen(bam_get_qname(bp->p)) + 1, (uint8_t*)bam_get_qname(bp->p))) {
- fprintf(samtools_stderr, "[markdup] error: unable to append 'do' tag.\n");
- return 1;
- }
+
+ if (check_chain) {
+ if (duplicate_chain_check(param, dup_hash, bp->p->duplicate, &opt_warnings, &single_optical, &optical))
+ goto fail;
}
- if (supp) {
- if (bam_aux_get(dup, "SA") || (dup->core.flag & BAM_FMUNMAP)) {
- if (add_duplicate(dup_hash, dup)) {
- return 1;
- }
- }
+ if (duplicate_chain_check(param, dup_hash, bp->p, &opt_warnings, &single_optical, &optical))
+ goto fail;
+
+
}
- }
single_dup++;
} else {
fprintf(samtools_stderr, "[markdup] error: single hashing failure.\n");
- return 1;
+ goto fail;
}
}
} else {
/* keep a moving window of reads based on coordinates and max read length. Any unaligned reads
should just be written as they cannot be matched as duplicates. */
- if (in_read->pos + max_length > prev_coord && in_read->b->core.tid == prev_tid && (prev_tid != -1 || prev_coord != -1)) {
+ if (in_read->pos + param->max_length > prev_coord && in_read->b->core.tid == prev_tid && (prev_tid != -1 || prev_coord != -1)) {
break;
}
- if (!remove_dups || !(in_read->b->core.flag & BAM_FDUP)) {
- if (supp) {
+ if (!param->remove_dups || !(in_read->b->core.flag & BAM_FDUP)) {
+ if (param->supp) {
if (tmp_file_write(&temp, in_read->b)) {
fprintf(samtools_stderr, "[markdup] error: writing temp output failed.\n");
- return 1;
+ goto fail;
}
} else {
- if (sam_write1(out, header, in_read->b) < 0) {
+ if (sam_write1(param->out, header, in_read->b) < 0) {
fprintf(samtools_stderr, "[markdup] error: writing output failed.\n");
- return 1;
+ goto fail;
}
}
// set the next one up for reading
in_read = kl_pushp(read_queue, read_buffer);
+ if (!in_read) {
+ fprintf(samtools_stderr, "[markdup] out of memory\n");
+ goto fail;
+ }
if ((in_read->b = bam_init1()) == NULL) {
fprintf(samtools_stderr, "[markdup] error: unable to allocate memory for alignment.\n");
- return 1;
+ goto fail;
}
}
if (ret < -1) {
fprintf(samtools_stderr, "[markdup] error: truncated input file.\n");
- return 1;
+ goto fail;
}
// write out the end of the list
in_read = &kl_val(rq);
if (bam_get_qname(in_read->b)) { // last entry will be blank
- if (!remove_dups || !(in_read->b->core.flag & BAM_FDUP)) {
- if (supp) {
+ if (!param->remove_dups || !(in_read->b->core.flag & BAM_FDUP)) {
+ if (param->supp) {
if (tmp_file_write(&temp, in_read->b)) {
fprintf(samtools_stderr, "[markdup] error: writing temp output failed.\n");
- return 1;
+ goto fail;
}
} else {
- if (sam_write1(out, header, in_read->b) < 0) {
+ if (sam_write1(param->out, header, in_read->b) < 0) {
fprintf(samtools_stderr, "[markdup] error: writing output failed.\n");
- return 1;
+ goto fail;
}
}
rq = kl_begin(read_buffer);
}
- if (supp) {
+ if (param->supp) {
bam1_t *b;
if (tmp_file_end_write(&temp)) {
fprintf(samtools_stderr, "[markdup] error: unable to end tmp writing.\n");
- return 1;
+ goto fail;
}
// read data from temp file and mark duplicate supplementary alignments
- if (tmp_file_begin_read(&temp, NULL)) {
- return 1;
+ if (tmp_file_begin_read(&temp)) {
+ goto fail;
}
b = bam_init1();
while ((ret = tmp_file_read(&temp, b)) > 0) {
- if ((b->core.flag & BAM_FSUPPLEMENTARY) || (b->core.flag & BAM_FUNMAP)) {
+ if ((b->core.flag & BAM_FSUPPLEMENTARY) || (b->core.flag & BAM_FUNMAP) || (b->core.flag & BAM_FSECONDARY)) {
+
k = kh_get(duplicates, dup_hash, bam_get_qname(b));
if (k != kh_end(dup_hash)) {
+
b->core.flag |= BAM_FDUP;
+ np_duplicate++;
+
+ if (param->tag && kh_val(dup_hash, k).name) {
+ if (bam_aux_append(b, "do", 'Z', strlen(kh_val(dup_hash, k).name) + 1, (uint8_t*)kh_val(dup_hash, k).name)) {
+ fprintf(samtools_stderr, "[markdup] error: unable to append supplementary 'do' tag.\n");
+ goto fail;
+ }
+ }
+
+ if (param->opt_dist) {
+ if (kh_val(dup_hash, k).type) {
+ bam_aux_append(b, "dt", 'Z', 3, (const uint8_t *)"SQ");
+ np_opt_duplicate++;
+ } else {
+ bam_aux_append(b, "dt", 'Z', 3, (const uint8_t *)"LB");
+ }
+ }
}
}
- if (!remove_dups || !(b->core.flag & BAM_FDUP)) {
- if (sam_write1(out, header, b) < 0) {
+ if (!param->remove_dups || !(b->core.flag & BAM_FDUP)) {
+ if (sam_write1(param->out, header, b) < 0) {
fprintf(samtools_stderr, "[markdup] error: writing final output failed.\n");
- return 1;
+ goto fail;
}
}
}
if (ret == -1) {
fprintf(samtools_stderr, "[markdup] error: failed to read tmp file.\n");
- return 1;
+ goto fail;
}
for (k = kh_begin(dup_hash); k != kh_end(dup_hash); ++k) {
if (kh_exist(dup_hash, k)) {
+ free(kh_val(dup_hash, k).name);
free((char *)kh_key(dup_hash, k));
+ kh_key(dup_hash, k) = NULL;
}
}
- tmp_file_destroy(&temp, b, 0);
- kh_destroy(duplicates, dup_hash);
+ tmp_file_destroy(&temp);
bam_destroy1(b);
}
- if (do_stats) {
- fprintf(samtools_stderr, "READ %d WRITTEN %d \n"
- "EXCLUDED %d EXAMINED %d\n"
- "PAIRED %d SINGLE %d\n"
- "DULPICATE PAIR %d DUPLICATE SINGLE %d\n"
- "DUPLICATE TOTAL %d\n", reading, writing, excluded, examined, pair, single,
- duplicate, single_dup, single_dup + duplicate);
+ if (opt_warnings) {
+ fprintf(samtools_stderr, "[markdup] warning: number of failed attempts to get coordinates from read names = %ld\n",
+ opt_warnings);
+ }
+
+ if (param->do_stats) {
+ FILE *fp;
+ int file_open = 0;
+ unsigned long els;
+
+ if (param->stats_file) {
+ if (NULL == (fp = fopen(param->stats_file, "w"))) {
+ fprintf(samtools_stderr, "[markdup] warning: cannot write stats to %s.\n", param->stats_file);
+ fp = samtools_stderr;
+ } else {
+ file_open = 1;
+ }
+ } else {
+ fp = samtools_stderr;
+ }
+
+ els = estimate_library_size(pair, duplicate - optical);
+
+ fprintf(fp,
+ "COMMAND: %s\n"
+ "READ: %ld\n"
+ "WRITTEN: %ld\n"
+ "EXCLUDED: %ld\n"
+ "EXAMINED: %ld\n"
+ "PAIRED: %ld\n"
+ "SINGLE: %ld\n"
+ "DUPLICATE PAIR: %ld\n"
+ "DUPLICATE SINGLE: %ld\n"
+ "DUPLICATE PAIR OPTICAL: %ld\n"
+ "DUPLICATE SINGLE OPTICAL: %ld\n"
+ "DUPLICATE NON PRIMARY: %ld\n"
+ "DUPLICATE NON PRIMARY OPTICAL: %ld\n"
+ "DUPLICATE PRIMARY TOTAL: %ld\n"
+ "DUPLICATE TOTAL: %ld\n"
+ "ESTIMATED_LIBRARY_SIZE: %ld\n", param->arg_list, reading, writing, excluded, examined, pair, single,
+ duplicate, single_dup, optical, single_optical, np_duplicate, np_opt_duplicate,
+ single_dup + duplicate, single_dup + duplicate + np_duplicate, els);
+
+ if (file_open) {
+ fclose(fp);
+ }
+ }
+
+ if (param->write_index) {
+ if (sam_idx_save(param->out) < 0) {
+ print_error_errno("markdup", "writing index failed");
+ goto fail;
+ }
}
kh_destroy(reads, pair_hash);
kh_destroy(reads, single_hash);
kl_destroy(read_queue, read_buffer);
- bam_hdr_destroy(header);
+ kh_destroy(duplicates, dup_hash);
+ sam_hdr_destroy(header);
return 0;
+
+ fail:
+ for (rq = kl_begin(read_buffer); rq != kl_end(read_buffer); rq = kl_next(rq))
+ bam_destroy1(kl_val(rq).b);
+ kl_destroy(read_queue, read_buffer);
+
+ for (k = kh_begin(dup_hash); k != kh_end(dup_hash); ++k) {
+ if (kh_exist(dup_hash, k)) {
+ free((char *)kh_key(dup_hash, k));
+ }
+ }
+ kh_destroy(duplicates, dup_hash);
+
+ kh_destroy(reads, pair_hash);
+ kh_destroy(reads, single_hash);
+ sam_hdr_destroy(header);
+ return 1;
}
fprintf(samtools_stderr, "\n");
fprintf(samtools_stderr, "Usage: samtools markdup <input.bam> <output.bam>\n\n");
fprintf(samtools_stderr, "Option: \n");
- fprintf(samtools_stderr, " -r Remove duplicate reads\n");
- fprintf(samtools_stderr, " -l INT Max read length (default 300 bases)\n");
- fprintf(samtools_stderr, " -S Mark supplemenary alignments of duplicates as duplicates (slower).\n");
- fprintf(samtools_stderr, " -s Report stats.\n");
- fprintf(samtools_stderr, " -T PREFIX Write temporary files to PREFIX.samtools.nnnn.nnnn.tmp.\n");
- fprintf(samtools_stderr, " -t Mark primary duplicates with the name of the original in a \'do\' tag."
+ fprintf(samtools_stderr, " -r Remove duplicate reads\n");
+ fprintf(samtools_stderr, " -l INT Max read length (default 300 bases)\n");
+ fprintf(samtools_stderr, " -S Mark supplementary alignments of duplicates as duplicates (slower).\n");
+ fprintf(samtools_stderr, " -s Report stats.\n");
+ fprintf(samtools_stderr, " -f NAME Write stats to named file. Implies -s.\n");
+ fprintf(samtools_stderr, " -T PREFIX Write temporary files to PREFIX.samtools.nnnn.nnnn.tmp.\n");
+ fprintf(samtools_stderr, " -d INT Optical distance (if set, marks with dt tag)\n");
+ fprintf(samtools_stderr, " -c Clear previous duplicate settings and tags.\n");
+ fprintf(samtools_stderr, " -m --mode TYPE Duplicate decision method for paired reads.\n"
+ " TYPE = t measure positions based on template start/end (default).\n"
+ " s measure positions based on sequence start.\n");
+ fprintf(samtools_stderr, " --include-fails Include quality check failed reads.\n");
+ fprintf(samtools_stderr, " --no-PG Do not add a PG line\n");
+ fprintf(samtools_stderr, " -t Mark primary duplicates with the name of the original in a \'do\' tag."
" Mainly for information and debugging.\n");
- sam_global_opt_help(samtools_stderr, "-.O..@");
+ sam_global_opt_help(samtools_stderr, "-.O..@..");
fprintf(samtools_stderr, "\nThe input file must be coordinate sorted and must have gone"
" through fixmates with the mate scoring option on.\n");
int bam_markdup(int argc, char **argv) {
- int c, ret, remove_dups = 0, report_stats = 0, include_supplementary = 0, tag_dup = 0;
- int32_t max_length = 300;
- samFile *in = NULL, *out = NULL;
+ int c, ret;
char wmode[3] = {'w', 'b', 0};
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
htsThreadPool p = {NULL, 0};
kstring_t tmpprefix = {0, 0, NULL};
struct stat st;
unsigned int t;
+ md_param_t param = {NULL, NULL, NULL, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, NULL, NULL, NULL};
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 0, '@'),
+ {"include-fails", no_argument, NULL, 1001},
+ {"no-PG", no_argument, NULL, 1002},
+ {"mode", required_argument, NULL, 'm'},
{NULL, 0, NULL, 0}
};
- while ((c = getopt_long(argc, argv, "rsl:StT:O:@:", lopts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "rsl:StT:O:@:f:d:ncm:", lopts, NULL)) >= 0) {
switch (c) {
- case 'r': remove_dups = 1; break;
- case 'l': max_length = atoi(optarg); break;
- case 's': report_stats = 1; break;
+ case 'r': param.remove_dups = 1; break;
+ case 'l': param.max_length = atoi(optarg); break;
+ case 's': param.do_stats = 1; break;
case 'T': kputs(optarg, &tmpprefix); break;
- case 'S': include_supplementary = 1; break;
- case 't': tag_dup = 1; break;
+ case 'S': param.supp = 1; break;
+ case 't': param.tag = 1; break;
+ case 'f': param.stats_file = optarg; param.do_stats = 1; break;
+ case 'd': param.opt_dist = atoi(optarg); break;
+ case 'c': param.clear = 1; break;
+ case 'm':
+ if (strcmp(optarg, "t") == 0) {
+ param.mode = MD_MODE_TEMPLATE;
+ } else if (strcmp(optarg, "s") == 0) {
+ param.mode = MD_MODE_SEQUENCE;
+ } else {
+ fprintf(samtools_stderr, "[markdup] error: unknown mode '%s'.\n", optarg);
+ return markdup_usage();
+ }
+
+ break;
+ case 1001: param.include_fails = 1; break;
+ case 1002: param.no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': return markdup_usage();
if (optind + 2 > argc)
return markdup_usage();
- in = sam_open_format(argv[optind], "r", &ga.in);
+ if (param.opt_dist < 0) param.opt_dist = 0;
+ if (param.max_length < 0) param.max_length = 300;
+
+ param.in = sam_open_format(argv[optind], "r", &ga.in);
- if (!in) {
+ if (!param.in) {
print_error_errno("markdup", "failed to open \"%s\" for input", argv[optind]);
return 1;
}
sam_open_mode(wmode + 1, argv[optind + 1], NULL);
- out = sam_open_format(argv[optind + 1], wmode, &ga.out);
+ param.out = sam_open_format(argv[optind + 1], wmode, &ga.out);
- if (!out) {
+ if (!param.out) {
print_error_errno("markdup", "failed to open \"%s\" for output", argv[optind + 1]);
return 1;
}
return 1;
}
- hts_set_opt(in, HTS_OPT_THREAD_POOL, &p);
- hts_set_opt(out, HTS_OPT_THREAD_POOL, &p);
+ hts_set_opt(param.in, HTS_OPT_THREAD_POOL, &p);
+ hts_set_opt(param.out, HTS_OPT_THREAD_POOL, &p);
}
// actual stuff happens here
t = ((unsigned) time(NULL)) ^ ((unsigned) clock());
ksprintf(&tmpprefix, "samtools.%d.%u.tmp", (int) getpid(), t % 10000);
+ param.prefix = tmpprefix.s;
+
+ param.arg_list = stringify_argv(argc + 1, argv - 1);
+ param.write_index = ga.write_index;
+ param.out_fn = argv[optind + 1];
- ret = bam_mark_duplicates(in, out, tmpprefix.s, remove_dups, max_length, report_stats, include_supplementary, tag_dup);
+ ret = bam_mark_duplicates(¶m);
- sam_close(in);
+ sam_close(param.in);
- if (sam_close(out) < 0) {
+ if (sam_close(param.out) < 0) {
fprintf(samtools_stderr, "[markdup] error closing output file\n");
ret = 1;
}
if (p.pool) hts_tpool_destroy(p.pool);
+ free(param.arg_list);
free(tmpprefix.s);
sam_global_args_free(&ga);
/* bam_mate.c -- fix mate pairing information and clean up flags.
- Copyright (C) 2009, 2011-2017 Genome Research Ltd.
+ Copyright (C) 2009, 2011-2017, 2019 Genome Research Ltd.
Portions copyright (C) 2011 Broad Institute.
Portions copyright (C) 2012 Peter Cock, The James Hutton Institute.
#include "htslib/sam.h"
#include "samtools.h"
+
+#define MD_MIN_QUALITY 15
+
/*
* This function calculates ct tag for two bams, it assumes they are from the same template and
* writes the tag to the first read in position terms.
static void bam_template_cigar(bam1_t *b1, bam1_t *b2, kstring_t *str)
{
bam1_t *swap;
- int i, end;
+ int i;
+ hts_pos_t end;
uint32_t *cigar;
str->l = 0;
if (b1->core.tid != b2->core.tid || b1->core.tid < 0 || b1->core.pos < 0 || b2->core.pos < 0 || b1->core.flag&BAM_FUNMAP || b2->core.flag&BAM_FUNMAP) return; // coordinateless or not on the same chr; skip
bam1_t* first = a;
bam1_t* second = b;
- int32_t a_pos = a->core.flag&BAM_FREVERSE ? bam_endpos(a) : a->core.pos;
- int32_t b_pos = b->core.flag&BAM_FREVERSE ? bam_endpos(b) : b->core.pos;
+ hts_pos_t a_pos = a->core.flag&BAM_FREVERSE ? bam_endpos(a) : a->core.pos;
+ hts_pos_t b_pos = b->core.flag&BAM_FREVERSE ? bam_endpos(b) : b->core.pos;
if (a_pos > b_pos) {
first = b;
second = a;
int i;
for (i = 0; i < b->core.l_qseq; i++) {
- if (qual[i] >= 15) score += qual[i];
+ if (qual[i] >= MD_MIN_QUALITY) score += qual[i];
}
return score;
}
// currently, this function ONLY works if each read has one hit
-static int bam_mating_core(samFile *in, samFile *out, int remove_reads, int proper_pair_check, int add_ct, int do_mate_scoring)
+static int bam_mating_core(samFile *in, samFile *out, int remove_reads, int proper_pair_check, int add_ct, int do_mate_scoring, char *arg_list, int no_pg)
{
- bam_hdr_t *header;
+ sam_hdr_t *header;
bam1_t *b[2] = { NULL, NULL };
- int curr, has_prev, pre_end = 0, cur_end = 0, result;
- kstring_t str;
+ int curr, has_prev, result;
+ hts_pos_t pre_end = 0, cur_end = 0;
+ kstring_t str = KS_INITIALIZE;
- str.l = str.m = 0; str.s = 0;
header = sam_hdr_read(in);
if (header == NULL) {
fprintf(stderr, "[bam_mating_core] ERROR: Couldn't read header\n");
return 1;
}
+
// Accept unknown, unsorted, or queryname sort order, but error on coordinate sorted.
- if ((header->l_text > 3) && (strncmp(header->text, "@HD", 3) == 0)) {
- char *p, *q;
- p = strstr(header->text, "\tSO:coordinate");
- q = strchr(header->text, '\n');
- // Looking for SO:coordinate within the @HD line only
- // (e.g. must ignore in a @CO comment line later in header)
- if ((p != 0) && (p < q)) {
- fprintf(stderr, "[bam_mating_core] ERROR: Coordinate sorted, require grouped/sorted by queryname.\n");
- goto fail;
- }
+ if (!sam_hdr_find_tag_hd(header, "SO", &str) && str.s && !strcmp(str.s, "coordinate")) {
+ fprintf(stderr, "[bam_mating_core] ERROR: Coordinate sorted, require grouped/sorted by queryname.\n");
+ goto fail;
}
+ ks_free(&str);
+
+ if (!no_pg && sam_hdr_add_pg(header, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))
+ goto fail;
+
if (sam_hdr_write(out, header) < 0) goto write_fail;
b[0] = bam_init1();
cur_end = bam_endpos(cur);
// Check cur_end isn't past the end of the contig we're on, if it is set the UNMAP'd flag
- if (cur_end > (int)header->target_len[cur->core.tid]) cur->core.flag |= BAM_FUNMAP;
+ if (cur_end > sam_hdr_tid2len(header, cur->core.tid)) cur->core.flag |= BAM_FUNMAP;
}
if (has_prev) { // do we have a pair of reads to examine?
if (strcmp(bam_get_qname(cur), bam_get_qname(pre)) == 0) { // identical pair name
if (pre->core.tid == cur->core.tid && !(cur->core.flag&(BAM_FUNMAP|BAM_FMUNMAP))
&& !(pre->core.flag&(BAM_FUNMAP|BAM_FMUNMAP))) // if safe set TLEN/ISIZE
{
- uint32_t cur5, pre5;
+ hts_pos_t cur5, pre5;
cur5 = (cur->core.flag&BAM_FREVERSE)? cur_end : cur->core.pos;
pre5 = (pre->core.flag&BAM_FREVERSE)? pre_end : pre->core.pos;
cur->core.isize = pre5 - cur5; pre->core.isize = cur5 - pre5;
if (sam_write1(out, header, pre) < 0) goto write_fail;
}
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
bam_destroy1(b[0]);
bam_destroy1(b[1]);
- free(str.s);
+ ks_free(&str);
return 0;
write_fail:
print_error_errno("fixmate", "Couldn't write to output file");
fail:
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
bam_destroy1(b[0]);
bam_destroy1(b[1]);
+ ks_free(&str);
return 1;
}
" -r Remove unmapped reads and secondary alignments\n"
" -p Disable FR proper pair check\n"
" -c Add template cigar ct tag\n"
-" -m Add mate score tag\n");
+" -m Add mate score tag\n"
+" --no-PG do not add a PG line\n");
- sam_global_opt_help(where, "-.O..@");
+ sam_global_opt_help(where, "-.O..@-.");
fprintf(where,
"\n"
{
htsThreadPool p = {NULL, 0};
samFile *in = NULL, *out = NULL;
- int c, remove_reads = 0, proper_pair_check = 1, add_ct = 0, res = 1, mate_score = 0;
+ int c, remove_reads = 0, proper_pair_check = 1, add_ct = 0, res = 1, mate_score = 0, no_pg = 0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
char wmode[3] = {'w', 'b', 0};
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 0, '@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
+ char *arg_list = NULL;
// parse args
if (argc == 1) { usage(stdout); return 0; }
case 'p': proper_pair_check = 0; break;
case 'c': add_ct = 1; break;
case 'm': mate_score = 1; break;
+ case 1: no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': usage(stderr); goto fail;
}
if (optind+1 >= argc) { usage(stderr); goto fail; }
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1)))
+ goto fail;
+
// init
if ((in = sam_open_format(argv[optind], "rb", &ga.in)) == NULL) {
print_error_errno("fixmate", "cannot open input file");
}
// run
- res = bam_mating_core(in, out, remove_reads, proper_pair_check, add_ct, mate_score);
+ res = bam_mating_core(in, out, remove_reads, proper_pair_check, add_ct, mate_score, arg_list, no_pg);
// cleanup
sam_close(in);
}
if (p.pool) hts_tpool_destroy(p.pool);
+ free(arg_list);
sam_global_args_free(&ga);
return res;
if (in) sam_close(in);
if (out) sam_close(out);
if (p.pool) hts_tpool_destroy(p.pool);
+ free(arg_list);
sam_global_args_free(&ga);
return 1;
}
/* bam_mate.c -- fix mate pairing information and clean up flags.
- Copyright (C) 2009, 2011-2017 Genome Research Ltd.
+ Copyright (C) 2009, 2011-2017, 2019 Genome Research Ltd.
Portions copyright (C) 2011 Broad Institute.
Portions copyright (C) 2012 Peter Cock, The James Hutton Institute.
#include "htslib/sam.h"
#include "samtools.h"
+
+#define MD_MIN_QUALITY 15
+
/*
* This function calculates ct tag for two bams, it assumes they are from the same template and
* writes the tag to the first read in position terms.
static void bam_template_cigar(bam1_t *b1, bam1_t *b2, kstring_t *str)
{
bam1_t *swap;
- int i, end;
+ int i;
+ hts_pos_t end;
uint32_t *cigar;
str->l = 0;
if (b1->core.tid != b2->core.tid || b1->core.tid < 0 || b1->core.pos < 0 || b2->core.pos < 0 || b1->core.flag&BAM_FUNMAP || b2->core.flag&BAM_FUNMAP) return; // coordinateless or not on the same chr; skip
bam1_t* first = a;
bam1_t* second = b;
- int32_t a_pos = a->core.flag&BAM_FREVERSE ? bam_endpos(a) : a->core.pos;
- int32_t b_pos = b->core.flag&BAM_FREVERSE ? bam_endpos(b) : b->core.pos;
+ hts_pos_t a_pos = a->core.flag&BAM_FREVERSE ? bam_endpos(a) : a->core.pos;
+ hts_pos_t b_pos = b->core.flag&BAM_FREVERSE ? bam_endpos(b) : b->core.pos;
if (a_pos > b_pos) {
first = b;
second = a;
int i;
for (i = 0; i < b->core.l_qseq; i++) {
- if (qual[i] >= 15) score += qual[i];
+ if (qual[i] >= MD_MIN_QUALITY) score += qual[i];
}
return score;
}
// currently, this function ONLY works if each read has one hit
-static int bam_mating_core(samFile *in, samFile *out, int remove_reads, int proper_pair_check, int add_ct, int do_mate_scoring)
+static int bam_mating_core(samFile *in, samFile *out, int remove_reads, int proper_pair_check, int add_ct, int do_mate_scoring, char *arg_list, int no_pg)
{
- bam_hdr_t *header;
+ sam_hdr_t *header;
bam1_t *b[2] = { NULL, NULL };
- int curr, has_prev, pre_end = 0, cur_end = 0, result;
- kstring_t str;
+ int curr, has_prev, result;
+ hts_pos_t pre_end = 0, cur_end = 0;
+ kstring_t str = KS_INITIALIZE;
- str.l = str.m = 0; str.s = 0;
header = sam_hdr_read(in);
if (header == NULL) {
fprintf(samtools_stderr, "[bam_mating_core] ERROR: Couldn't read header\n");
return 1;
}
+
// Accept unknown, unsorted, or queryname sort order, but error on coordinate sorted.
- if ((header->l_text > 3) && (strncmp(header->text, "@HD", 3) == 0)) {
- char *p, *q;
- p = strstr(header->text, "\tSO:coordinate");
- q = strchr(header->text, '\n');
- // Looking for SO:coordinate within the @HD line only
- // (e.g. must ignore in a @CO comment line later in header)
- if ((p != 0) && (p < q)) {
- fprintf(samtools_stderr, "[bam_mating_core] ERROR: Coordinate sorted, require grouped/sorted by queryname.\n");
- goto fail;
- }
+ if (!sam_hdr_find_tag_hd(header, "SO", &str) && str.s && !strcmp(str.s, "coordinate")) {
+ fprintf(samtools_stderr, "[bam_mating_core] ERROR: Coordinate sorted, require grouped/sorted by queryname.\n");
+ goto fail;
}
+ ks_free(&str);
+
+ if (!no_pg && sam_hdr_add_pg(header, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))
+ goto fail;
+
if (sam_hdr_write(out, header) < 0) goto write_fail;
b[0] = bam_init1();
cur_end = bam_endpos(cur);
// Check cur_end isn't past the end of the contig we're on, if it is set the UNMAP'd flag
- if (cur_end > (int)header->target_len[cur->core.tid]) cur->core.flag |= BAM_FUNMAP;
+ if (cur_end > sam_hdr_tid2len(header, cur->core.tid)) cur->core.flag |= BAM_FUNMAP;
}
if (has_prev) { // do we have a pair of reads to examine?
if (strcmp(bam_get_qname(cur), bam_get_qname(pre)) == 0) { // identical pair name
if (pre->core.tid == cur->core.tid && !(cur->core.flag&(BAM_FUNMAP|BAM_FMUNMAP))
&& !(pre->core.flag&(BAM_FUNMAP|BAM_FMUNMAP))) // if safe set TLEN/ISIZE
{
- uint32_t cur5, pre5;
+ hts_pos_t cur5, pre5;
cur5 = (cur->core.flag&BAM_FREVERSE)? cur_end : cur->core.pos;
pre5 = (pre->core.flag&BAM_FREVERSE)? pre_end : pre->core.pos;
cur->core.isize = pre5 - cur5; pre->core.isize = cur5 - pre5;
if (sam_write1(out, header, pre) < 0) goto write_fail;
}
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
bam_destroy1(b[0]);
bam_destroy1(b[1]);
- free(str.s);
+ ks_free(&str);
return 0;
write_fail:
print_error_errno("fixmate", "Couldn't write to output file");
fail:
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
bam_destroy1(b[0]);
bam_destroy1(b[1]);
+ ks_free(&str);
return 1;
}
" -r Remove unmapped reads and secondary alignments\n"
" -p Disable FR proper pair check\n"
" -c Add template cigar ct tag\n"
-" -m Add mate score tag\n");
+" -m Add mate score tag\n"
+" --no-PG do not add a PG line\n");
- sam_global_opt_help(where, "-.O..@");
+ sam_global_opt_help(where, "-.O..@-.");
fprintf(where,
"\n"
{
htsThreadPool p = {NULL, 0};
samFile *in = NULL, *out = NULL;
- int c, remove_reads = 0, proper_pair_check = 1, add_ct = 0, res = 1, mate_score = 0;
+ int c, remove_reads = 0, proper_pair_check = 1, add_ct = 0, res = 1, mate_score = 0, no_pg = 0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
char wmode[3] = {'w', 'b', 0};
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 0, '@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
+ char *arg_list = NULL;
// parse args
if (argc == 1) { usage(samtools_stdout); return 0; }
case 'p': proper_pair_check = 0; break;
case 'c': add_ct = 1; break;
case 'm': mate_score = 1; break;
+ case 1: no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': usage(samtools_stderr); goto fail;
}
if (optind+1 >= argc) { usage(samtools_stderr); goto fail; }
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1)))
+ goto fail;
+
// init
if ((in = sam_open_format(argv[optind], "rb", &ga.in)) == NULL) {
print_error_errno("fixmate", "cannot open input file");
}
// run
- res = bam_mating_core(in, out, remove_reads, proper_pair_check, add_ct, mate_score);
+ res = bam_mating_core(in, out, remove_reads, proper_pair_check, add_ct, mate_score, arg_list, no_pg);
// cleanup
sam_close(in);
}
if (p.pool) hts_tpool_destroy(p.pool);
+ free(arg_list);
sam_global_args_free(&ga);
return res;
if (in) sam_close(in);
if (out) sam_close(out);
if (p.pool) hts_tpool_destroy(p.pool);
+ free(arg_list);
sam_global_args_free(&ga);
return 1;
}
/* bam_md.c -- calmd subcommand.
- Copyright (C) 2009-2011, 2014-2015 Genome Research Ltd.
+ Copyright (C) 2009-2011, 2014-2015, 2019 Genome Research Ltd.
Portions copyright (C) 2009-2011 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
int bam_aux_drop_other(bam1_t *b, uint8_t *s);
-void bam_fillmd1_core(bam1_t *b, char *ref, int ref_len, int flag, int max_nm, int quiet_mode)
+void bam_fillmd1_core(bam1_t *b, char *ref, hts_pos_t ref_len, int flag, int max_nm, int quiet_mode)
{
uint8_t *seq = bam_get_seq(b);
uint32_t *cigar = bam_get_cigar(b);
bam1_core_t *c = &b->core;
- int i, x, y, u = 0;
+ int i, y, u = 0;
+ hts_pos_t x;
kstring_t *str;
int32_t old_nm_i = -1, nm = 0;
if (flag&USE_EQUAL) seq[z/2] &= (z&1)? 0xf0 : 0x0f;
++u;
} else {
- kputw(u, str); kputc(ref[x+j], str);
+ kputw(u, str); kputc(toupper(ref[x+j]), str);
u = 0; ++nm;
}
}
kputw(u, str); kputc('^', str);
for (j = 0; j < l; ++j) {
if (x+j >= ref_len || ref[x+j] == '\0') break;
- kputc(ref[x+j], str);
+ kputc(toupper(ref[x+j]), str);
}
u = 0;
x += j; nm += j;
" -A modify the quality string\n"
" -Q use quiet mode to output less debug info to stdout\n"
" -r compute the BQ tag (without -A) or cap baseQ by BAQ (with -A)\n"
-" -E extended BAQ for better sensitivity but lower specificity\n");
+" -E extended BAQ for better sensitivity but lower specificity\n"
+" --no-PG do not add a PG line\n");
- sam_global_opt_help(stderr, "-....@");
+ sam_global_opt_help(stderr, "-....@-.");
return 1;
}
int bam_fillmd(int argc, char *argv[])
{
- int c, flt_flag, tid = -2, ret, len, is_bam_out, is_uncompressed, max_nm, is_realn, capQ, baq_flag, quiet_mode;
+ int c, flt_flag, tid = -2, ret, is_bam_out, is_uncompressed, max_nm, is_realn, capQ, baq_flag, quiet_mode, no_pg = 0;
+ hts_pos_t len;
htsThreadPool p = {NULL, 0};
samFile *fp = NULL, *fpout = NULL;
- bam_hdr_t *header = NULL;
+ sam_hdr_t *header = NULL;
faidx_t *fai = NULL;
- char *ref = NULL, mode_w[8], *ref_file;
+ char *ref = NULL, mode_w[8], *ref_file, *arg_list = NULL;
bam1_t *b = NULL;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 0,'@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
case 'A': baq_flag |= 1; break;
case 'E': baq_flag |= 2; break;
case 'Q': quiet_mode = 1; break;
+ case 1: no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
fprintf(stderr, "[bam_fillmd] unrecognized option '-%c'\n\n", c);
/* else fall-through */
return 1;
}
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("calmd", "failed to create arg_list");
+ return 1;
+ }
+
header = sam_hdr_read(fp);
- if (header == NULL || header->n_targets == 0) {
+ if (header == NULL || sam_hdr_nref(header) == 0) {
fprintf(stderr, "[bam_fillmd] input SAM does not have header. Abort!\n");
goto fail;
}
print_error_errno("calmd", "Failed to open output");
goto fail;
}
+ if (!no_pg && sam_hdr_add_pg(header, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error("calmd", "failed to add PG line to header");
+ goto fail;
+ }
if (sam_hdr_write(fpout, header) < 0) {
print_error_errno("calmd", "Failed to write sam header");
goto fail;
if (b->core.tid >= 0) {
if (tid != b->core.tid) {
free(ref);
- ref = fai_fetch(fai, header->target_name[b->core.tid], &len);
+ ref = fai_fetch64(fai, sam_hdr_tid2name(header, b->core.tid), &len);
tid = b->core.tid;
if (ref == 0) { // FIXME: Should this always be fatal?
fprintf(stderr, "[bam_fillmd] fail to find sequence '%s' in the reference.\n",
- header->target_name[tid]);
+ sam_hdr_tid2name(header, tid));
if (is_realn || capQ > 10) goto fail; // Would otherwise crash
}
}
goto fail;
}
bam_destroy1(b);
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
+ free(arg_list);
free(ref);
fai_destroy(fai);
sam_close(fp);
return 0;
fail:
+ free(arg_list);
free(ref);
if (b) bam_destroy1(b);
- if (header) bam_hdr_destroy(header);
+ if (header) sam_hdr_destroy(header);
if (fai) fai_destroy(fai);
if (fp) sam_close(fp);
if (fpout) sam_close(fpout);
/* bam_md.c -- calmd subcommand.
- Copyright (C) 2009-2011, 2014-2015 Genome Research Ltd.
+ Copyright (C) 2009-2011, 2014-2015, 2019 Genome Research Ltd.
Portions copyright (C) 2009-2011 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
int bam_aux_drop_other(bam1_t *b, uint8_t *s);
-void bam_fillmd1_core(bam1_t *b, char *ref, int ref_len, int flag, int max_nm, int quiet_mode)
+void bam_fillmd1_core(bam1_t *b, char *ref, hts_pos_t ref_len, int flag, int max_nm, int quiet_mode)
{
uint8_t *seq = bam_get_seq(b);
uint32_t *cigar = bam_get_cigar(b);
bam1_core_t *c = &b->core;
- int i, x, y, u = 0;
+ int i, y, u = 0;
+ hts_pos_t x;
kstring_t *str;
int32_t old_nm_i = -1, nm = 0;
if (flag&USE_EQUAL) seq[z/2] &= (z&1)? 0xf0 : 0x0f;
++u;
} else {
- kputw(u, str); kputc(ref[x+j], str);
+ kputw(u, str); kputc(toupper(ref[x+j]), str);
u = 0; ++nm;
}
}
kputw(u, str); kputc('^', str);
for (j = 0; j < l; ++j) {
if (x+j >= ref_len || ref[x+j] == '\0') break;
- kputc(ref[x+j], str);
+ kputc(toupper(ref[x+j]), str);
}
u = 0;
x += j; nm += j;
" -A modify the quality string\n"
" -Q use quiet mode to output less debug info to samtools_stdout\n"
" -r compute the BQ tag (without -A) or cap baseQ by BAQ (with -A)\n"
-" -E extended BAQ for better sensitivity but lower specificity\n");
+" -E extended BAQ for better sensitivity but lower specificity\n"
+" --no-PG do not add a PG line\n");
- sam_global_opt_help(samtools_stderr, "-....@");
+ sam_global_opt_help(samtools_stderr, "-....@-.");
return 1;
}
int bam_fillmd(int argc, char *argv[])
{
- int c, flt_flag, tid = -2, ret, len, is_bam_out, is_uncompressed, max_nm, is_realn, capQ, baq_flag, quiet_mode;
+ int c, flt_flag, tid = -2, ret, is_bam_out, is_uncompressed, max_nm, is_realn, capQ, baq_flag, quiet_mode, no_pg = 0;
+ hts_pos_t len;
htsThreadPool p = {NULL, 0};
samFile *fp = NULL, *fpout = NULL;
- bam_hdr_t *header = NULL;
+ sam_hdr_t *header = NULL;
faidx_t *fai = NULL;
- char *ref = NULL, mode_w[8], *ref_file;
+ char *ref = NULL, mode_w[8], *ref_file, *arg_list = NULL;
bam1_t *b = NULL;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 0,'@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
case 'A': baq_flag |= 1; break;
case 'E': baq_flag |= 2; break;
case 'Q': quiet_mode = 1; break;
+ case 1: no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
fprintf(samtools_stderr, "[bam_fillmd] unrecognized option '-%c'\n\n", c);
/* else fall-through */
return 1;
}
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("calmd", "failed to create arg_list");
+ return 1;
+ }
+
header = sam_hdr_read(fp);
- if (header == NULL || header->n_targets == 0) {
+ if (header == NULL || sam_hdr_nref(header) == 0) {
fprintf(samtools_stderr, "[bam_fillmd] input SAM does not have header. Abort!\n");
goto fail;
}
print_error_errno("calmd", "Failed to open output");
goto fail;
}
+ if (!no_pg && sam_hdr_add_pg(header, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error("calmd", "failed to add PG line to header");
+ goto fail;
+ }
if (sam_hdr_write(fpout, header) < 0) {
print_error_errno("calmd", "Failed to write sam header");
goto fail;
if (b->core.tid >= 0) {
if (tid != b->core.tid) {
free(ref);
- ref = fai_fetch(fai, header->target_name[b->core.tid], &len);
+ ref = fai_fetch64(fai, sam_hdr_tid2name(header, b->core.tid), &len);
tid = b->core.tid;
if (ref == 0) { // FIXME: Should this always be fatal?
fprintf(samtools_stderr, "[bam_fillmd] fail to find sequence '%s' in the reference.\n",
- header->target_name[tid]);
+ sam_hdr_tid2name(header, tid));
if (is_realn || capQ > 10) goto fail; // Would otherwise crash
}
}
goto fail;
}
bam_destroy1(b);
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
+ free(arg_list);
free(ref);
fai_destroy(fai);
sam_close(fp);
return 0;
fail:
+ free(arg_list);
free(ref);
if (b) bam_destroy1(b);
- if (header) bam_hdr_destroy(header);
+ if (header) sam_hdr_destroy(header);
if (fai) fai_destroy(fai);
if (fp) sam_close(fp);
if (fpout) sam_close(fpout);
int bam_plbuf_push(const bam1_t *b, bam_plbuf_t *buf)
{
- int ret, n_plp, tid, pos;
+ int ret, n_plp, tid;
+ hts_pos_t pos;
const bam_pileup1_t *plp;
ret = bam_plp_push(buf->iter, b);
if (ret < 0) return ret;
- while ((plp = bam_plp_next(buf->iter, &tid, &pos, &n_plp)) != 0)
+ while ((plp = bam_plp64_next(buf->iter, &tid, &pos, &n_plp)) != 0)
buf->func(tid, pos, n_plp, plp, buf->data);
return 0;
}
int bam_plbuf_push(const bam1_t *b, bam_plbuf_t *buf)
{
- int ret, n_plp, tid, pos;
+ int ret, n_plp, tid;
+ hts_pos_t pos;
const bam_pileup1_t *plp;
ret = bam_plp_push(buf->iter, b);
if (ret < 0) return ret;
- while ((plp = bam_plp_next(buf->iter, &tid, &pos, &n_plp)) != 0)
+ while ((plp = bam_plp64_next(buf->iter, &tid, &pos, &n_plp)) != 0)
buf->func(tid, pos, n_plp, plp, buf->data);
return 0;
}
#ifndef BAM_PILEUP_F_DEFINED
#define BAM_PILEUP_F_DEFINED
-typedef int (*bam_pileup_f)(uint32_t tid, uint32_t pos, int n, const bam_pileup1_t *pl, void *data);
+typedef int (*bam_pileup_f)(uint32_t tid, hts_pos_t pos, int n, const bam_pileup1_t *pl, void *data);
#endif //BAM_PILEUP_F_DEFINED
typedef struct {
/* bam_plcmd.c -- mpileup subcommand.
- Copyright (C) 2008-2015 Genome Research Ltd.
+ Copyright (C) 2008-2015, 2019 Genome Research Ltd.
Portions copyright (C) 2009-2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include <errno.h>
#include <sys/stat.h>
#include <getopt.h>
+#include <inttypes.h>
#include <htslib/sam.h>
#include <htslib/faidx.h>
#include <htslib/kstring.h>
+#include <htslib/klist.h>
#include <htslib/khash_str2int.h>
-#include "sam_header.h"
#include "samtools.h"
+#include "bedidx.h"
#include "sam_opts.h"
+#define dummy_free(p)
+KLIST_INIT(auxlist, char *, dummy_free)
+
static inline int printw(int c, FILE *fp)
{
char buf[16];
return 0;
}
-static inline void pileup_seq(FILE *fp, const bam_pileup1_t *p, int pos, int ref_len, const char *ref)
+static inline int pileup_seq(FILE *fp, const bam_pileup1_t *p, hts_pos_t pos,
+ hts_pos_t ref_len, const char *ref, kstring_t *ks,
+ int rev_del)
{
int j;
if (p->is_head) {
else c = bam_is_rev(p->b)? tolower(c) : toupper(c);
}
putc(c, fp);
- } else putc(p->is_refskip? (bam_is_rev(p->b)? '<' : '>') : '*', fp);
+ } else putc(p->is_refskip? (bam_is_rev(p->b)? '<' : '>') : ((bam_is_rev(p->b) && rev_del) ? '#' : '*'), fp);
+ int del_len = -p->indel;
if (p->indel > 0) {
- putc('+', fp); printw(p->indel, fp);
- for (j = 1; j <= p->indel; ++j) {
- int c = seq_nt16_str[bam_seqi(bam_get_seq(p->b), p->qpos + j)];
- putc(bam_is_rev(p->b)? tolower(c) : toupper(c), fp);
+ int len = bam_plp_insertion(p, ks, &del_len);
+ if (len < 0)
+ return -1;
+ putc('+', fp); printw(len, fp);
+ if (bam_is_rev(p->b)) {
+ char pad = rev_del ? '#' : '*';
+ for (j = 0; j < len; j++)
+ putc(ks->s[j] != '*' ? tolower(ks->s[j]) : pad, fp);
+ } else {
+ for (j = 0; j < len; j++)
+ putc(toupper(ks->s[j]), fp);
}
- } else if (p->indel < 0) {
- printw(p->indel, fp);
- for (j = 1; j <= -p->indel; ++j) {
+ }
+ if (del_len > 0) {
+ printw(-del_len, fp);
+ for (j = 1; j <= del_len; ++j) {
int c = (ref && (int)pos+j < ref_len)? ref[pos+j] : 'N';
putc(bam_is_rev(p->b)? tolower(c) : toupper(c), fp);
}
}
if (p->is_tail) putc('$', fp);
+ return 0;
}
#include <assert.h>
#define MPLP_REDO_BAQ (1<<6)
#define MPLP_ILLUMINA13 (1<<7)
#define MPLP_IGNORE_RG (1<<8)
-#define MPLP_PRINT_POS (1<<9)
-#define MPLP_PRINT_MAPQ (1<<10)
+#define MPLP_PRINT_QPOS (1<<9)
#define MPLP_PER_SAMPLE (1<<11)
#define MPLP_SMART_OVERLAPS (1<<12)
+
#define MPLP_PRINT_QNAME (1<<13)
+#define MPLP_PRINT_FLAG (1<<14)
+#define MPLP_PRINT_RNAME (1<<15)
+#define MPLP_PRINT_POS (1<<16)
+#define MPLP_PRINT_MAPQ (1<<17)
+#define MPLP_PRINT_CIGAR (1<<18)
+#define MPLP_PRINT_RNEXT (1<<19)
+#define MPLP_PRINT_PNEXT (1<<20)
+#define MPLP_PRINT_TLEN (1<<21)
+#define MPLP_PRINT_SEQ (1<<22)
+#define MPLP_PRINT_QUAL (1<<23)
#define MPLP_MAX_DEPTH 8000
#define MPLP_MAX_INDEL_DEPTH 250
-void *bed_read(const char *fn);
-void bed_destroy(void *_h);
-int bed_overlap(const void *_h, const char *chr, int beg, int end);
-
typedef struct {
- int min_mq, flag, min_baseQ, capQ_thres, max_depth, max_indel_depth, fmt_flag, all;
+ int min_mq, flag, min_baseQ, capQ_thres, max_depth, max_indel_depth, fmt_flag, all, rev_del;
int rflag_require, rflag_filter;
int openQ, extQ, tandemQ, min_support; // for indels
double min_frac; // for indels
char *reg, *pl_list, *fai_fname, *output_fname;
faidx_t *fai;
- void *bed, *rghash;
+ void *bed, *rghash, *auxlist;
int argc;
char **argv;
+ char sep, empty;
sam_global_args ga;
} mplp_conf_t;
typedef struct {
char *ref[2];
int ref_id[2];
- int ref_len[2];
+ hts_pos_t ref_len[2];
} mplp_ref_t;
#define MPLP_REF_INIT {{NULL,NULL},{-1,-1},{0,0}}
typedef struct {
samFile *fp;
hts_itr_t *iter;
- bam_hdr_t *h;
+ sam_hdr_t *h;
mplp_ref_t *ref;
const mplp_conf_t *conf;
} mplp_aux_t;
bam_pileup1_t **plp;
} mplp_pileup_t;
-static int mplp_get_ref(mplp_aux_t *ma, int tid, char **ref, int *ref_len) {
+static int build_auxlist(mplp_conf_t *conf, char *optstring) {
+ if (!optstring)
+ return 0;
+
+ void *colhash = khash_str2int_init();
+ if (!colhash)
+ return 1;
+
+ struct active_cols {
+ char *name;
+ int supported;
+ };
+
+ const struct active_cols colnames[11] = {
+ {"QNAME", 1}, {"FLAG", 1}, {"RNAME", 1}, {"POS", 1}, {"MAPQ", 1}, {"CIGAR", 0}, {"RNEXT", 1}, {"PNEXT", 1}, {"TLEN", 0}, {"SEQ", 0}, {"QUAL", 0}
+ };
+
+ int i, f = MPLP_PRINT_QNAME, colno = 11;
+ for (i = 0; i < colno; i++, f <<= 1)
+ if (colnames[i].supported)
+ khash_str2int_set(colhash, colnames[i].name, f);
+
+ conf->auxlist = kl_init(auxlist);
+ if (!conf->auxlist)
+ return 1;
+
+ char *save_p;
+ char *tag = strtok_r(optstring, ",", &save_p);
+ while (tag) {
+ if (khash_str2int_get(colhash, tag, &f) == 0) {
+ conf->flag |= f;
+ } else {
+ if (strlen(tag) != 2) {
+ fprintf(stderr, "[%s] tag '%s' has more than two characters or not supported\n", __func__, tag);
+ } else {
+ char **tag_p = kl_pushp(auxlist, conf->auxlist);
+ *tag_p = tag;
+ }
+ }
+ tag = strtok_r(NULL, ",", &save_p);
+ }
+
+ khash_str2int_destroy(colhash);
+
+ return 0;
+}
+
+static int mplp_get_ref(mplp_aux_t *ma, int tid, char **ref, hts_pos_t *ref_len) {
mplp_ref_t *r = ma->ref;
//printf("get ref %d {%d/%p, %d/%p}\n", tid, r->ref_id[0], r->ref[0], r->ref_id[1], r->ref[1]);
}
if (tid == r->ref_id[1]) {
// Last, swap over
- int tmp;
- tmp = r->ref_id[0]; r->ref_id[0] = r->ref_id[1]; r->ref_id[1] = tmp;
- tmp = r->ref_len[0]; r->ref_len[0] = r->ref_len[1]; r->ref_len[1] = tmp;
+ int tmp_id;
+ hts_pos_t tmp_len;
+ tmp_id = r->ref_id[0]; r->ref_id[0] = r->ref_id[1]; r->ref_id[1] = tmp_id;
+ tmp_len = r->ref_len[0]; r->ref_len[0] = r->ref_len[1]; r->ref_len[1] = tmp_len;
char *tc;
tc = r->ref[0]; r->ref[0] = r->ref[1]; r->ref[1] = tc;
r->ref_len[1] = r->ref_len[0];
r->ref_id[0] = tid;
- r->ref[0] = faidx_fetch_seq(ma->conf->fai,
- ma->h->target_name[r->ref_id[0]],
+ r->ref[0] = faidx_fetch_seq64(ma->conf->fai,
+ sam_hdr_tid2name(ma->h, r->ref_id[0]),
0,
- INT_MAX,
+ HTS_POS_MAX,
&r->ref_len[0]);
if (!r->ref[0]) {
static void
print_empty_pileup(FILE *fp, const mplp_conf_t *conf, const char *tname,
- int pos, int n, const char *ref, int ref_len)
+ hts_pos_t pos, int n, const char *ref, hts_pos_t ref_len)
{
int i;
- fprintf(fp, "%s\t%d\t%c", tname, pos+1, (ref && pos < ref_len)? ref[pos] : 'N');
+ fprintf(fp, "%s\t%"PRIhts_pos"\t%c", tname, pos+1, (ref && pos < ref_len)? ref[pos] : 'N');
for (i = 0; i < n; ++i) {
fputs("\t0\t*\t*", fp);
- if (conf->flag & MPLP_PRINT_MAPQ) fputs("\t*", fp);
- if (conf->flag & MPLP_PRINT_POS) fputs("\t*", fp);
- if (conf->flag & MPLP_PRINT_QNAME) fputs("\t*", fp);
+ if (conf->flag & MPLP_PRINT_QPOS)
+ fputs("\t*", fp);
+ int flag_value = MPLP_PRINT_QNAME;
+ while(flag_value < MPLP_PRINT_QUAL + 1) {
+ if (conf->flag & flag_value)
+ fputs("\t*", fp);
+ flag_value <<= 1;
+ }
+ if (conf->auxlist) {
+ int t = 0;
+ while(t++ < ((klist_t(auxlist) *)conf->auxlist)->size)
+ fputs("\t*", fp);
+ }
}
putc('\n', fp);
}
{
char *ref;
mplp_aux_t *ma = (mplp_aux_t*)data;
- int ret, skip = 0, ref_len;
+ int ret, skip = 0;
+ hts_pos_t ref_len;
+
do {
int has_ref;
ret = ma->iter? sam_itr_next(ma->fp, ma->iter, b) : sam_read1(ma->fp, ma->h, b);
if (ma->conf->rflag_require && !(ma->conf->rflag_require&b->core.flag)) { skip = 1; continue; }
if (ma->conf->rflag_filter && ma->conf->rflag_filter&b->core.flag) { skip = 1; continue; }
if (ma->conf->bed && ma->conf->all == 0) { // test overlap
- skip = !bed_overlap(ma->conf->bed, ma->h->target_name[b->core.tid], b->core.pos, bam_endpos(b));
+ skip = !bed_overlap(ma->conf->bed, sam_hdr_tid2name(ma->h, b->core.tid), b->core.pos, bam_endpos(b));
if (skip) continue;
}
if (ma->conf->rghash) { // exclude read groups
if (ma->conf->fai && b->core.tid >= 0) {
has_ref = mplp_get_ref(ma, b->core.tid, &ref, &ref_len);
if (has_ref && ref_len <= b->core.pos) { // exclude reads outside of the reference sequence
- fprintf(stderr,"[%s] Skipping because %d is outside of %d [ref:%d]\n",
- __func__, b->core.pos, ref_len, b->core.tid);
+ fprintf(stderr,"[%s] Skipping because %"PRIhts_pos" is outside of %"PRIhts_pos" [ref:%d]\n",
+ __func__, (int64_t) b->core.pos, ref_len, b->core.tid);
skip = 1;
continue;
}
* @param conf configuration for this pileup
* @param n number of files specified in fn
* @param fn filenames
+ * @param fn_idx index filenames
*/
-static int mpileup(mplp_conf_t *conf, int n, char **fn)
+static int mpileup(mplp_conf_t *conf, int n, char **fn, char **fn_idx)
{
extern void *bcf_call_add_rg(void *rghash, const char *hdtext, const char *list);
extern void bcf_call_del_rghash(void *rghash);
mplp_aux_t **data;
- int i, tid, pos, *n_plp, beg0 = 0, end0 = INT_MAX, tid0 = 0, ref_len, max_depth, max_indel_depth;
+ int i, tid, *n_plp, tid0 = 0, max_depth, max_indel_depth;
+ hts_pos_t pos, beg0 = 0, end0 = HTS_POS_MAX, ref_len;
const bam_pileup1_t **plp;
mplp_ref_t mp_ref = MPLP_REF_INIT;
bam_mplp_t iter;
- bam_hdr_t *h = NULL; /* header of first file in input list */
+ sam_hdr_t *h = NULL; /* header of first file in input list */
char *ref;
void *rghash = NULL;
FILE *pileup_fp = NULL;
// read the header of each file in the list and initialize data
for (i = 0; i < n; ++i) {
- bam_hdr_t *h_tmp;
+ sam_hdr_t *h_tmp;
data[i] = calloc(1, sizeof(mplp_aux_t));
data[i]->fp = sam_open_format(fn[i], "rb", &conf->ga.in);
if ( !data[i]->fp )
fprintf(stderr,"[%s] fail to read the header of %s\n", __func__, fn[i]);
exit(EXIT_FAILURE);
}
- bam_smpl_add(sm, fn[i], (conf->flag&MPLP_IGNORE_RG)? 0 : h_tmp->text);
+ bam_smpl_add(sm, fn[i], (conf->flag&MPLP_IGNORE_RG)? 0 : sam_hdr_str(h_tmp));
if (conf->flag & MPLP_BCF) {
// Collect read group IDs with PL (platform) listed in pl_list (note: fragile, strstr search)
- rghash = bcf_call_add_rg(rghash, h_tmp->text, conf->pl_list);
+ rghash = bcf_call_add_rg(rghash, sam_hdr_str(h_tmp), conf->pl_list);
}
if (conf->reg) {
- hts_idx_t *idx = sam_index_load(data[i]->fp, fn[i]);
+ hts_idx_t *idx = NULL;
+ // If index filename has not been specfied, look in BAM folder
+ if (fn_idx != NULL) {
+ idx = sam_index_load2(data[i]->fp, fn[i], fn_idx[i]);
+ } else {
+ idx = sam_index_load(data[i]->fp, fn[i]);
+ }
+
if (idx == NULL) {
fprintf(stderr, "[%s] fail to load index for %s\n", __func__, fn[i]);
exit(EXIT_FAILURE);
if (i == 0) h = data[i]->h = h_tmp; // save the header of the first file
else {
// FIXME: check consistency between h and h_tmp
- bam_hdr_destroy(h_tmp);
+ sam_hdr_destroy(h_tmp);
// we store only the first file's header; it's (alleged to be)
// compatible with the i-th file's target_name lookup needs
// Translate BAM @SQ tags to BCF ##contig tags
// todo: use/write new BAM header manipulation routines, fill also UR, M5
- for (i=0; i<h->n_targets; i++)
+ for (i=0; i < sam_hdr_nref(h); i++)
{
str.l = 0;
- ksprintf(&str, "##contig=<ID=%s,length=%d>", h->target_name[i], h->target_len[i]);
+ ksprintf(&str, "##contig=<ID=%s,length=%"PRId64">", sam_hdr_tid2name(h, i), (int64_t) sam_hdr_tid2len(h, i));
bcf_hdr_append(bcf_hdr, str.s);
}
free(str.s);
for (i=0; i<sm->n; i++)
bcf_hdr_add_sample(bcf_hdr, sm->smpl[i]);
bcf_hdr_add_sample(bcf_hdr, NULL);
- bcf_hdr_write(bcf_fp, bcf_hdr);
+ if (bcf_hdr_write(bcf_fp, bcf_hdr) != 0) {
+ print_error_errno("mpileup", "Failed to write VCF/BCF header to \"%s\"",
+ conf->output_fname? conf->output_fname : "standard output");
+ exit(EXIT_FAILURE);
+ }
// End of BCF header creation
// Initialise the calling algorithm
bam_mplp_set_maxcnt(iter, max_depth);
bcf1_t *bcf_rec = bcf_init1();
int ret;
- int last_tid = -1, last_pos = -1;
+ int last_tid = -1;
+ hts_pos_t last_pos = -1;
// begin pileup
- while ( (ret=bam_mplp_auto(iter, &tid, &pos, n_plp, plp)) > 0) {
+ while ( (ret=bam_mplp64_auto(iter, &tid, &pos, n_plp, plp)) > 0) {
if (conf->reg && (pos < beg0 || pos >= end0)) continue; // out of the region requested
mplp_get_ref(data[0], tid, &ref, &ref_len);
//printf("tid=%d len=%d ref=%p/%s\n", tid, ref_len, ref, ref);
if (conf->flag & MPLP_BCF) {
int total_depth, _ref0, ref16;
- if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, h->target_name[tid], pos, pos+1)) continue;
+ if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, sam_hdr_tid2name(h, tid), pos, pos+1)) continue;
for (i = total_depth = 0; i < n; ++i) total_depth += n_plp[i];
group_smpl(&gplp, sm, &buf, n, fn, n_plp, plp, conf->flag & MPLP_IGNORE_RG);
_ref0 = (ref && pos < ref_len)? ref[pos] : 'N';
bcf_call_combine(gplp.n, bcr, bca, ref16, &bc);
bcf_clear1(bcf_rec);
bcf_call2bcf(&bc, bcf_rec, bcr, conf->fmt_flag, 0, 0);
- bcf_write1(bcf_fp, bcf_hdr, bcf_rec);
+ if (bcf_write1(bcf_fp, bcf_hdr, bcf_rec) != 0) {
+ print_error_errno("mpileup", "Failed to write VCF/BCF record to \"%s\"",
+ conf->output_fname?conf->output_fname:"standard output");
+ exit(EXIT_FAILURE);
+ }
// call indels; todo: subsampling with total_depth>max_indel_depth instead of ignoring?
if (!(conf->flag&MPLP_NO_INDEL) && total_depth < max_indel_depth && bcf_call_gap_prep(gplp.n, gplp.n_plp, gplp.plp, pos, bca, ref, rghash) >= 0)
{
if (bcf_call_combine(gplp.n, bcr, bca, -1, &bc) >= 0) {
bcf_clear1(bcf_rec);
bcf_call2bcf(&bc, bcf_rec, bcr, conf->fmt_flag, bca, ref);
- bcf_write1(bcf_fp, bcf_hdr, bcf_rec);
+ if (bcf_write1(bcf_fp, bcf_hdr, bcf_rec) != 0) {
+ print_error_errno("mpileup", "Failed to write VCF/BCF record to \"%s\"",
+ conf->output_fname?conf->output_fname:"standard output");
+ exit(EXIT_FAILURE);
+ }
}
}
} else {
// Deal with missing portions of previous tids
while (tid > last_tid) {
if (last_tid >= 0 && !conf->reg) {
- while (++last_pos < h->target_len[last_tid]) {
- if (conf->bed && bed_overlap(conf->bed, h->target_name[last_tid], last_pos, last_pos + 1) == 0)
+ while (++last_pos < sam_hdr_tid2len(h, last_tid)) {
+ if (conf->bed && bed_overlap(conf->bed, sam_hdr_tid2name(h, last_tid), last_pos, last_pos + 1) == 0)
continue;
- print_empty_pileup(pileup_fp, conf, h->target_name[last_tid], last_pos, n, ref, ref_len);
+ print_empty_pileup(pileup_fp, conf, sam_hdr_tid2name(h, last_tid), last_pos, n, ref, ref_len);
}
}
last_tid++;
// Deal with missing portion of current tid
while (++last_pos < pos) {
if (conf->reg && last_pos < beg0) continue; // out of range; skip
- if (conf->bed && bed_overlap(conf->bed, h->target_name[tid], last_pos, last_pos + 1) == 0)
+ if (conf->bed && bed_overlap(conf->bed, sam_hdr_tid2name(h, tid), last_pos, last_pos + 1) == 0)
continue;
- print_empty_pileup(pileup_fp, conf, h->target_name[tid], last_pos, n, ref, ref_len);
+ print_empty_pileup(pileup_fp, conf, sam_hdr_tid2name(h, tid), last_pos, n, ref, ref_len);
}
last_tid = tid;
last_pos = pos;
}
- if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, h->target_name[tid], pos, pos+1)) continue;
+ if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, sam_hdr_tid2name(h, tid), pos, pos+1)) continue;
- fprintf(pileup_fp, "%s\t%d\t%c", h->target_name[tid], pos + 1, (ref && pos < ref_len)? ref[pos] : 'N');
+ fprintf(pileup_fp, "%s\t%"PRIhts_pos"\t%c", sam_hdr_tid2name(h, tid), pos + 1, (ref && pos < ref_len)? ref[pos] : 'N');
for (i = 0; i < n; ++i) {
int j, cnt;
for (j = cnt = 0; j < n_plp[i]; ++j) {
fprintf(pileup_fp, "\t%d\t", cnt);
if (n_plp[i] == 0) {
fputs("*\t*", pileup_fp);
- if (conf->flag & MPLP_PRINT_MAPQ) fputs("\t*", pileup_fp);
- if (conf->flag & MPLP_PRINT_POS) fputs("\t*", pileup_fp);
- if (conf->flag & MPLP_PRINT_QNAME) fputs("\t*", pileup_fp);
+ if (conf->flag & MPLP_PRINT_QPOS)
+ fputs("\t*", pileup_fp);
+ int flag_value = MPLP_PRINT_QNAME;
+ while(flag_value < MPLP_PRINT_QUAL + 1) {
+ if (conf->flag & flag_value)
+ fputs("\t*", pileup_fp);
+ flag_value <<= 1;
+ }
+ if (conf->auxlist) {
+ int t = 0;
+ while(t++ < ((klist_t(auxlist) *)conf->auxlist)->size)
+ fputs("\t*", pileup_fp);
+ }
} else {
int n = 0;
+ kstring_t ks = KS_INITIALIZE;
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
int c = p->qpos < p->b->core.l_qseq
? bam_get_qual(p->b)[p->qpos]
: 0;
- if (c >= conf->min_baseQ)
- n++, pileup_seq(pileup_fp, plp[i] + j, pos, ref_len, ref);
+ if (c >= conf->min_baseQ) {
+ n++;
+ if (pileup_seq(pileup_fp, plp[i] + j, pos, ref_len, ref, &ks, conf->rev_del) < 0) {
+ ret = 1;
+ goto fail;
+ }
+ }
}
if (!n) putc('*', pileup_fp);
+ /* Print base qualities */
n = 0;
+ ks_free(&ks);
putc('\t', pileup_fp);
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
}
if (!n) putc('*', pileup_fp);
- if (conf->flag & MPLP_PRINT_MAPQ) {
+ /* Print mpileup positions */
+ if (conf->flag & MPLP_PRINT_QPOS) {
n = 0;
putc('\t', pileup_fp);
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
int c = p->qpos < p->b->core.l_qseq
- ? bam_get_qual(p->b)[p->qpos]
- : 0;
+ ? bam_get_qual(p->b)[p->qpos]
+ : 0;
if ( c < conf->min_baseQ ) continue;
- c = plp[i][j].b->core.qual + 33;
- if (c > 126) c = 126;
- putc(c, pileup_fp);
+ if (n > 0) putc(',', pileup_fp);
n++;
+ fprintf(pileup_fp, "%d", p->qpos + 1);
}
if (!n) putc('*', pileup_fp);
}
- if (conf->flag & MPLP_PRINT_POS) {
- n = 0;
- putc('\t', pileup_fp);
- for (j = 0; j < n_plp[i]; ++j) {
- const bam_pileup1_t *p = plp[i] + j;
- int c = p->qpos < p->b->core.l_qseq
- ? bam_get_qual(p->b)[p->qpos]
- : 0;
- if ( c < conf->min_baseQ ) continue;
-
- if (n > 0) putc(',', pileup_fp);
- fprintf(pileup_fp, "%d", plp[i][j].qpos + 1); // FIXME: printf() is very slow...
- n++;
+ /* Print selected columns */
+ int flag_value = MPLP_PRINT_QNAME;
+ while(flag_value < MPLP_PRINT_QUAL + 1) {
+ if (conf->flag & flag_value) {
+ n = 0;
+ putc('\t', pileup_fp);
+ for (j = 0; j < n_plp[i]; ++j) {
+ const bam_pileup1_t *p = &plp[i][j];
+ int c = p->qpos < p->b->core.l_qseq
+ ? bam_get_qual(p->b)[p->qpos]
+ : 0;
+ if ( c < conf->min_baseQ ) continue;
+ if (n > 0 && flag_value != MPLP_PRINT_MAPQ) putc(',', pileup_fp);
+ n++;
+
+ switch (flag_value) {
+ case MPLP_PRINT_QNAME:
+ fputs(bam_get_qname(p->b), pileup_fp);
+ break;
+ case MPLP_PRINT_FLAG:
+ fprintf(pileup_fp, "%d", p->b->core.flag);
+ break;
+ case MPLP_PRINT_RNAME:
+ if (p->b->core.tid >= 0)
+ fputs(sam_hdr_tid2name(h, p->b->core.tid), pileup_fp);
+ else
+ putc('*', pileup_fp);
+ break;
+ case MPLP_PRINT_POS:
+ fprintf(pileup_fp, "%"PRId64, (int64_t) p->b->core.pos + 1);
+ break;
+ case MPLP_PRINT_MAPQ:
+ c = p->b->core.qual + 33;
+ if (c > 126) c = 126;
+ putc(c, pileup_fp);
+ break;
+ case MPLP_PRINT_RNEXT:
+ if (p->b->core.mtid >= 0)
+ fputs(sam_hdr_tid2name(h, p->b->core.mtid), pileup_fp);
+ else
+ putc('*', pileup_fp);
+ break;
+ case MPLP_PRINT_PNEXT:
+ fprintf(pileup_fp, "%"PRId64, (int64_t) p->b->core.mpos + 1);
+ break;
+ }
+ }
+ if (!n) putc('*', pileup_fp);
}
- if (!n) putc('*', pileup_fp);
+ flag_value <<= 1;
}
- if (conf->flag & MPLP_PRINT_QNAME) {
- n = 0;
- putc('\t', pileup_fp);
- for (j = 0; j < n_plp[i]; ++j) {
- const bam_pileup1_t *p = &plp[i][j];
- int c = p->qpos < p->b->core.l_qseq
- ? bam_get_qual(p->b)[p->qpos]
- : 0;
- if ( c < conf->min_baseQ ) continue;
-
- if (n > 0) putc(',', pileup_fp);
- fputs(bam_get_qname(p->b), pileup_fp);
- n++;
+ /* Print selected tags */
+ klist_t(auxlist) *auxlist_p = ((klist_t(auxlist) *)conf->auxlist);
+ if (auxlist_p && auxlist_p->size) {
+ kliter_t(auxlist) *aux;
+ for (aux = kl_begin(auxlist_p); aux != kl_end(auxlist_p); aux = kl_next(aux)) {
+ n = 0;
+ putc('\t', pileup_fp);
+ for (j = 0; j < n_plp[i]; ++j) {
+ const bam_pileup1_t *p = &plp[i][j];
+ int c = p->qpos < p->b->core.l_qseq
+ ? bam_get_qual(p->b)[p->qpos]
+ : 0;
+ if ( c < conf->min_baseQ ) continue;
+
+ if (n > 0) putc(conf->sep, pileup_fp);
+ n++;
+ uint8_t* tag_u = bam_aux_get(p->b, kl_val(aux));
+ if (!tag_u) {
+ putc(conf->empty , pileup_fp);
+ continue;
+ }
+
+ /* Tag value is string */
+ if (*tag_u == 'Z' || *tag_u == 'H') {
+ char *tag_s = bam_aux2Z(tag_u);
+ if (!tag_s) continue;
+ fputs(tag_s, pileup_fp);
+ }
+
+ /* Tag value is integer */
+ if (*tag_u == 'I' || *tag_u == 'i' || *tag_u == 'C' || *tag_u == 'c' || *tag_u == 'S' || *tag_u == 's') {
+ int64_t tag_i = bam_aux2i(tag_u);
+ fprintf(pileup_fp, "%" PRId64 "", tag_i);
+ }
+
+ /* Tag value is float */
+ if (*tag_u == 'd' || *tag_u == 'f') {
+ double tag_f = bam_aux2f(tag_u);
+ fprintf(pileup_fp, "%lf", tag_f);
+ }
+
+ /* Tag value is character */
+ if (*tag_u == 'A') {
+ char tag_c = bam_aux2A(tag_u);
+ putc(tag_c, pileup_fp);
+ }
+ }
+ if (!n) putc('*', pileup_fp);
}
- if (!n) putc('*', pileup_fp);
}
}
}
last_pos = beg0-1;
mplp_get_ref(data[0], tid0, &ref, &ref_len);
}
- while (last_tid >= 0 && last_tid < h->n_targets) {
- while (++last_pos < h->target_len[last_tid]) {
+ while (last_tid >= 0 && last_tid < sam_hdr_nref(h)) {
+ while (++last_pos < sam_hdr_tid2len(h, last_tid)) {
if (last_pos >= end0) break;
- if (conf->bed && bed_overlap(conf->bed, h->target_name[last_tid], last_pos, last_pos + 1) == 0)
+ if (conf->bed && bed_overlap(conf->bed, sam_hdr_tid2name(h, last_tid), last_pos, last_pos + 1) == 0)
continue;
- print_empty_pileup(pileup_fp, conf, h->target_name[last_tid], last_pos, n, ref, ref_len);
+ print_empty_pileup(pileup_fp, conf, sam_hdr_tid2name(h, last_tid), last_pos, n, ref, ref_len);
}
last_tid++;
last_pos = -1;
}
}
+fail:
// clean up
free(bc.tmp.s);
bcf_destroy1(bcf_rec);
free(gplp.plp); free(gplp.n_plp); free(gplp.m_plp);
bcf_call_del_rghash(rghash);
bam_mplp_destroy(iter);
- bam_hdr_destroy(h);
+ sam_hdr_destroy(h);
for (i = 0; i < n; ++i) {
sam_close(data[i]->fp);
if (data[i]->iter) hts_itr_destroy(data[i]->iter);
" [%s]\n", tmp_filter);
fprintf(fp,
" -x, --ignore-overlaps disable read-pair overlap detection\n"
+" -X, --customized-index use customized index files\n" // -X flag for index filename
"\n"
"Output options:\n"
" -o, --output FILE write output to FILE [standard output]\n"
" -O, --output-BP output base positions on reads\n"
" -s, --output-MQ output mapping quality\n"
" --output-QNAME output read names\n"
+" --output-extra STR output extra read fields and read tag values\n"
+" --output-sep CHAR set the separator character for tag lists [,]\n"
+" --output-empty CHAR set the no value character for tag lists [*]\n"
+" --reverse-del use '#' character for deletions on the reverse strand\n"
" -a output all positions (including zero depth)\n"
" -a -a (or -aa) output absolutely all positions, including unused ref. sequences\n"
"\n"
"Generic options:\n");
- sam_global_opt_help(fp, "-.--.-");
+ sam_global_opt_help(fp, "-.--.--.");
fprintf(fp, "\n"
"Note that using \"samtools mpileup\" to generate BCF or VCF files is now\n"
int c;
const char *file_list = NULL;
char **fn = NULL;
- int nfiles = 0, use_orphan = 0;
+ int nfiles = 0, use_orphan = 0, has_index_file = 0;
mplp_conf_t mplp;
memset(&mplp, 0, sizeof(mplp_conf_t));
mplp.min_baseQ = 13;
mplp.rflag_filter = BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP;
mplp.output_fname = NULL;
mplp.all = 0;
+ mplp.rev_del = 0;
+ mplp.sep = ',';
+ mplp.empty = '*';
sam_global_args_init(&mplp.ga);
static const struct option lopts[] =
{"per-sample-mF", no_argument, NULL, 'p'},
{"per-sample-mf", no_argument, NULL, 'p'},
{"platforms", required_argument, NULL, 'P'},
+ {"customized-index", no_argument, NULL, 'X'},
+ {"reverse-del", no_argument, NULL, 6},
+ {"output-extra", required_argument, NULL, 7},
+ {"output-sep", required_argument, NULL, 8},
+ {"output-empty", required_argument, NULL, 9},
{NULL, 0, NULL, 0}
};
- while ((c = getopt_long(argc, argv, "Agf:r:l:q:Q:uRC:BDSd:L:b:P:po:e:h:Im:F:EG:6OsVvxt:a",lopts,NULL)) >= 0) {
+
+ while ((c = getopt_long(argc, argv, "Agf:r:l:q:Q:uRC:BDSd:L:b:P:po:e:h:Im:F:EG:6OsVvxXt:a",lopts,NULL)) >= 0) {
switch (c) {
case 'x': mplp.flag &= ~MPLP_SMART_OVERLAPS; break;
case 1 :
case 3 : mplp.output_fname = optarg; break;
case 4 : mplp.openQ = atoi(optarg); break;
case 5 : mplp.flag |= MPLP_PRINT_QNAME; break;
+ case 6 : mplp.rev_del = 1; break;
+ case 7 :
+ if (build_auxlist(&mplp, optarg) != 0) {
+ fprintf(stderr,"Could not build aux list using '%s'\n", optarg);
+ return 1;
+ }
+ break;
+ case 8: mplp.sep = optarg[0]; break;
+ case 9: mplp.empty = optarg[0]; break;
case 'f':
mplp.fai = fai_load(optarg);
if (mplp.fai == NULL) return 1;
case 'v': mplp.flag |= MPLP_BCF | MPLP_VCF; deprecated(c); break;
case 'u': mplp.flag |= MPLP_NO_COMP | MPLP_BCF; deprecated(c); break;
case 'B': mplp.flag &= ~MPLP_REALN; break;
+ case 'X': has_index_file = 1; break;
case 'D': mplp.fmt_flag |= B2B_FMT_DP; deprecated(c); break;
case 'S': mplp.fmt_flag |= B2B_FMT_SP; deprecated(c); break;
case 'V': mplp.fmt_flag |= B2B_FMT_DV; deprecated(c); break;
case '6': mplp.flag |= MPLP_ILLUMINA13; break;
case 'R': mplp.flag |= MPLP_IGNORE_RG; break;
case 's': mplp.flag |= MPLP_PRINT_MAPQ; break;
- case 'O': mplp.flag |= MPLP_PRINT_POS; break;
+ case 'O': mplp.flag |= MPLP_PRINT_QPOS; break;
case 'C': mplp.capQ_thres = atoi(optarg); break;
case 'q': mplp.min_mq = atoi(optarg); break;
case 'Q': mplp.min_baseQ = atoi(optarg); break;
}
int ret;
if (file_list) {
+ if (has_index_file) {
+ fprintf(stderr,"Error: The -b option cannot be combined with -X\n"); // No customize index loc in file list mode
+ return 1;
+ }
if ( read_file_list(file_list,&nfiles,&fn) ) return 1;
- ret = mpileup(&mplp,nfiles,fn);
+ ret = mpileup(&mplp,nfiles,fn,NULL);
for (c=0; c<nfiles; c++) free(fn[c]);
free(fn);
}
- else
- ret = mpileup(&mplp, argc - optind, argv + optind);
+ else {
+ if (has_index_file) {
+ if ((argc - optind)%2 !=0) { // Calculate # of input BAM files
+ fprintf(stderr, "Odd number of filenames detected! Each BAM file should have an index file\n");
+ return 1;
+ }
+ nfiles = (argc - optind)/2;
+ ret = mpileup(&mplp, nfiles, argv + optind, argv + nfiles + optind);
+ } else {
+ nfiles = argc - optind;
+ ret = mpileup(&mplp, nfiles, argv + optind, NULL);
+ }
+ }
if (mplp.rghash) khash_str2int_destroy_free(mplp.rghash);
free(mplp.reg); free(mplp.pl_list);
if (mplp.fai) fai_destroy(mplp.fai);
if (mplp.bed) bed_destroy(mplp.bed);
+ if (mplp.auxlist) kl_destroy(auxlist, (klist_t(auxlist) *)mplp.auxlist);
return ret;
}
/* bam_plcmd.c -- mpileup subcommand.
- Copyright (C) 2008-2015 Genome Research Ltd.
+ Copyright (C) 2008-2015, 2019 Genome Research Ltd.
Portions copyright (C) 2009-2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include <errno.h>
#include <sys/stat.h>
#include <getopt.h>
+#include <inttypes.h>
#include <htslib/sam.h>
#include <htslib/faidx.h>
#include <htslib/kstring.h>
+#include <htslib/klist.h>
#include <htslib/khash_str2int.h>
-#include "sam_header.h"
#include "samtools.h"
+#include "bedidx.h"
#include "sam_opts.h"
+#define dummy_free(p)
+KLIST_INIT(auxlist, char *, dummy_free)
+
static inline int printw(int c, FILE *fp)
{
char buf[16];
return 0;
}
-static inline void pileup_seq(FILE *fp, const bam_pileup1_t *p, int pos, int ref_len, const char *ref)
+static inline int pileup_seq(FILE *fp, const bam_pileup1_t *p, hts_pos_t pos,
+ hts_pos_t ref_len, const char *ref, kstring_t *ks,
+ int rev_del)
{
int j;
if (p->is_head) {
else c = bam_is_rev(p->b)? tolower(c) : toupper(c);
}
putc(c, fp);
- } else putc(p->is_refskip? (bam_is_rev(p->b)? '<' : '>') : '*', fp);
+ } else putc(p->is_refskip? (bam_is_rev(p->b)? '<' : '>') : ((bam_is_rev(p->b) && rev_del) ? '#' : '*'), fp);
+ int del_len = -p->indel;
if (p->indel > 0) {
- putc('+', fp); printw(p->indel, fp);
- for (j = 1; j <= p->indel; ++j) {
- int c = seq_nt16_str[bam_seqi(bam_get_seq(p->b), p->qpos + j)];
- putc(bam_is_rev(p->b)? tolower(c) : toupper(c), fp);
+ int len = bam_plp_insertion(p, ks, &del_len);
+ if (len < 0)
+ return -1;
+ putc('+', fp); printw(len, fp);
+ if (bam_is_rev(p->b)) {
+ char pad = rev_del ? '#' : '*';
+ for (j = 0; j < len; j++)
+ putc(ks->s[j] != '*' ? tolower(ks->s[j]) : pad, fp);
+ } else {
+ for (j = 0; j < len; j++)
+ putc(toupper(ks->s[j]), fp);
}
- } else if (p->indel < 0) {
- printw(p->indel, fp);
- for (j = 1; j <= -p->indel; ++j) {
+ }
+ if (del_len > 0) {
+ printw(-del_len, fp);
+ for (j = 1; j <= del_len; ++j) {
int c = (ref && (int)pos+j < ref_len)? ref[pos+j] : 'N';
putc(bam_is_rev(p->b)? tolower(c) : toupper(c), fp);
}
}
if (p->is_tail) putc('$', fp);
+ return 0;
}
#include <assert.h>
#define MPLP_REDO_BAQ (1<<6)
#define MPLP_ILLUMINA13 (1<<7)
#define MPLP_IGNORE_RG (1<<8)
-#define MPLP_PRINT_POS (1<<9)
-#define MPLP_PRINT_MAPQ (1<<10)
+#define MPLP_PRINT_QPOS (1<<9)
#define MPLP_PER_SAMPLE (1<<11)
#define MPLP_SMART_OVERLAPS (1<<12)
+
#define MPLP_PRINT_QNAME (1<<13)
+#define MPLP_PRINT_FLAG (1<<14)
+#define MPLP_PRINT_RNAME (1<<15)
+#define MPLP_PRINT_POS (1<<16)
+#define MPLP_PRINT_MAPQ (1<<17)
+#define MPLP_PRINT_CIGAR (1<<18)
+#define MPLP_PRINT_RNEXT (1<<19)
+#define MPLP_PRINT_PNEXT (1<<20)
+#define MPLP_PRINT_TLEN (1<<21)
+#define MPLP_PRINT_SEQ (1<<22)
+#define MPLP_PRINT_QUAL (1<<23)
#define MPLP_MAX_DEPTH 8000
#define MPLP_MAX_INDEL_DEPTH 250
-void *bed_read(const char *fn);
-void bed_destroy(void *_h);
-int bed_overlap(const void *_h, const char *chr, int beg, int end);
-
typedef struct {
- int min_mq, flag, min_baseQ, capQ_thres, max_depth, max_indel_depth, fmt_flag, all;
+ int min_mq, flag, min_baseQ, capQ_thres, max_depth, max_indel_depth, fmt_flag, all, rev_del;
int rflag_require, rflag_filter;
int openQ, extQ, tandemQ, min_support; // for indels
double min_frac; // for indels
char *reg, *pl_list, *fai_fname, *output_fname;
faidx_t *fai;
- void *bed, *rghash;
+ void *bed, *rghash, *auxlist;
int argc;
char **argv;
+ char sep, empty;
sam_global_args ga;
} mplp_conf_t;
typedef struct {
char *ref[2];
int ref_id[2];
- int ref_len[2];
+ hts_pos_t ref_len[2];
} mplp_ref_t;
#define MPLP_REF_INIT {{NULL,NULL},{-1,-1},{0,0}}
typedef struct {
samFile *fp;
hts_itr_t *iter;
- bam_hdr_t *h;
+ sam_hdr_t *h;
mplp_ref_t *ref;
const mplp_conf_t *conf;
} mplp_aux_t;
bam_pileup1_t **plp;
} mplp_pileup_t;
-static int mplp_get_ref(mplp_aux_t *ma, int tid, char **ref, int *ref_len) {
+static int build_auxlist(mplp_conf_t *conf, char *optstring) {
+ if (!optstring)
+ return 0;
+
+ void *colhash = khash_str2int_init();
+ if (!colhash)
+ return 1;
+
+ struct active_cols {
+ char *name;
+ int supported;
+ };
+
+ const struct active_cols colnames[11] = {
+ {"QNAME", 1}, {"FLAG", 1}, {"RNAME", 1}, {"POS", 1}, {"MAPQ", 1}, {"CIGAR", 0}, {"RNEXT", 1}, {"PNEXT", 1}, {"TLEN", 0}, {"SEQ", 0}, {"QUAL", 0}
+ };
+
+ int i, f = MPLP_PRINT_QNAME, colno = 11;
+ for (i = 0; i < colno; i++, f <<= 1)
+ if (colnames[i].supported)
+ khash_str2int_set(colhash, colnames[i].name, f);
+
+ conf->auxlist = kl_init(auxlist);
+ if (!conf->auxlist)
+ return 1;
+
+ char *save_p;
+ char *tag = strtok_r(optstring, ",", &save_p);
+ while (tag) {
+ if (khash_str2int_get(colhash, tag, &f) == 0) {
+ conf->flag |= f;
+ } else {
+ if (strlen(tag) != 2) {
+ fprintf(samtools_stderr, "[%s] tag '%s' has more than two characters or not supported\n", __func__, tag);
+ } else {
+ char **tag_p = kl_pushp(auxlist, conf->auxlist);
+ *tag_p = tag;
+ }
+ }
+ tag = strtok_r(NULL, ",", &save_p);
+ }
+
+ khash_str2int_destroy(colhash);
+
+ return 0;
+}
+
+static int mplp_get_ref(mplp_aux_t *ma, int tid, char **ref, hts_pos_t *ref_len) {
mplp_ref_t *r = ma->ref;
//printf("get ref %d {%d/%p, %d/%p}\n", tid, r->ref_id[0], r->ref[0], r->ref_id[1], r->ref[1]);
}
if (tid == r->ref_id[1]) {
// Last, swap over
- int tmp;
- tmp = r->ref_id[0]; r->ref_id[0] = r->ref_id[1]; r->ref_id[1] = tmp;
- tmp = r->ref_len[0]; r->ref_len[0] = r->ref_len[1]; r->ref_len[1] = tmp;
+ int tmp_id;
+ hts_pos_t tmp_len;
+ tmp_id = r->ref_id[0]; r->ref_id[0] = r->ref_id[1]; r->ref_id[1] = tmp_id;
+ tmp_len = r->ref_len[0]; r->ref_len[0] = r->ref_len[1]; r->ref_len[1] = tmp_len;
char *tc;
tc = r->ref[0]; r->ref[0] = r->ref[1]; r->ref[1] = tc;
r->ref_len[1] = r->ref_len[0];
r->ref_id[0] = tid;
- r->ref[0] = faidx_fetch_seq(ma->conf->fai,
- ma->h->target_name[r->ref_id[0]],
+ r->ref[0] = faidx_fetch_seq64(ma->conf->fai,
+ sam_hdr_tid2name(ma->h, r->ref_id[0]),
0,
- INT_MAX,
+ HTS_POS_MAX,
&r->ref_len[0]);
if (!r->ref[0]) {
static void
print_empty_pileup(FILE *fp, const mplp_conf_t *conf, const char *tname,
- int pos, int n, const char *ref, int ref_len)
+ hts_pos_t pos, int n, const char *ref, hts_pos_t ref_len)
{
int i;
- fprintf(fp, "%s\t%d\t%c", tname, pos+1, (ref && pos < ref_len)? ref[pos] : 'N');
+ fprintf(fp, "%s\t%"PRIhts_pos"\t%c", tname, pos+1, (ref && pos < ref_len)? ref[pos] : 'N');
for (i = 0; i < n; ++i) {
fputs("\t0\t*\t*", fp);
- if (conf->flag & MPLP_PRINT_MAPQ) fputs("\t*", fp);
- if (conf->flag & MPLP_PRINT_POS) fputs("\t*", fp);
- if (conf->flag & MPLP_PRINT_QNAME) fputs("\t*", fp);
+ if (conf->flag & MPLP_PRINT_QPOS)
+ fputs("\t*", fp);
+ int flag_value = MPLP_PRINT_QNAME;
+ while(flag_value < MPLP_PRINT_QUAL + 1) {
+ if (conf->flag & flag_value)
+ fputs("\t*", fp);
+ flag_value <<= 1;
+ }
+ if (conf->auxlist) {
+ int t = 0;
+ while(t++ < ((klist_t(auxlist) *)conf->auxlist)->size)
+ fputs("\t*", fp);
+ }
}
putc('\n', fp);
}
{
char *ref;
mplp_aux_t *ma = (mplp_aux_t*)data;
- int ret, skip = 0, ref_len;
+ int ret, skip = 0;
+ hts_pos_t ref_len;
+
do {
int has_ref;
ret = ma->iter? sam_itr_next(ma->fp, ma->iter, b) : sam_read1(ma->fp, ma->h, b);
if (ma->conf->rflag_require && !(ma->conf->rflag_require&b->core.flag)) { skip = 1; continue; }
if (ma->conf->rflag_filter && ma->conf->rflag_filter&b->core.flag) { skip = 1; continue; }
if (ma->conf->bed && ma->conf->all == 0) { // test overlap
- skip = !bed_overlap(ma->conf->bed, ma->h->target_name[b->core.tid], b->core.pos, bam_endpos(b));
+ skip = !bed_overlap(ma->conf->bed, sam_hdr_tid2name(ma->h, b->core.tid), b->core.pos, bam_endpos(b));
if (skip) continue;
}
if (ma->conf->rghash) { // exclude read groups
if (ma->conf->fai && b->core.tid >= 0) {
has_ref = mplp_get_ref(ma, b->core.tid, &ref, &ref_len);
if (has_ref && ref_len <= b->core.pos) { // exclude reads outside of the reference sequence
- fprintf(samtools_stderr,"[%s] Skipping because %d is outside of %d [ref:%d]\n",
- __func__, b->core.pos, ref_len, b->core.tid);
+ fprintf(samtools_stderr,"[%s] Skipping because %"PRIhts_pos" is outside of %"PRIhts_pos" [ref:%d]\n",
+ __func__, (int64_t) b->core.pos, ref_len, b->core.tid);
skip = 1;
continue;
}
* @param conf configuration for this pileup
* @param n number of files specified in fn
* @param fn filenames
+ * @param fn_idx index filenames
*/
-static int mpileup(mplp_conf_t *conf, int n, char **fn)
+static int mpileup(mplp_conf_t *conf, int n, char **fn, char **fn_idx)
{
extern void *bcf_call_add_rg(void *rghash, const char *hdtext, const char *list);
extern void bcf_call_del_rghash(void *rghash);
mplp_aux_t **data;
- int i, tid, pos, *n_plp, beg0 = 0, end0 = INT_MAX, tid0 = 0, ref_len, max_depth, max_indel_depth;
+ int i, tid, *n_plp, tid0 = 0, max_depth, max_indel_depth;
+ hts_pos_t pos, beg0 = 0, end0 = HTS_POS_MAX, ref_len;
const bam_pileup1_t **plp;
mplp_ref_t mp_ref = MPLP_REF_INIT;
bam_mplp_t iter;
- bam_hdr_t *h = NULL; /* header of first file in input list */
+ sam_hdr_t *h = NULL; /* header of first file in input list */
char *ref;
void *rghash = NULL;
FILE *pileup_fp = NULL;
// read the header of each file in the list and initialize data
for (i = 0; i < n; ++i) {
- bam_hdr_t *h_tmp;
+ sam_hdr_t *h_tmp;
data[i] = calloc(1, sizeof(mplp_aux_t));
data[i]->fp = sam_open_format(fn[i], "rb", &conf->ga.in);
if ( !data[i]->fp )
fprintf(samtools_stderr,"[%s] fail to read the header of %s\n", __func__, fn[i]);
exit(EXIT_FAILURE);
}
- bam_smpl_add(sm, fn[i], (conf->flag&MPLP_IGNORE_RG)? 0 : h_tmp->text);
+ bam_smpl_add(sm, fn[i], (conf->flag&MPLP_IGNORE_RG)? 0 : sam_hdr_str(h_tmp));
if (conf->flag & MPLP_BCF) {
// Collect read group IDs with PL (platform) listed in pl_list (note: fragile, strstr search)
- rghash = bcf_call_add_rg(rghash, h_tmp->text, conf->pl_list);
+ rghash = bcf_call_add_rg(rghash, sam_hdr_str(h_tmp), conf->pl_list);
}
if (conf->reg) {
- hts_idx_t *idx = sam_index_load(data[i]->fp, fn[i]);
+ hts_idx_t *idx = NULL;
+ // If index filename has not been specfied, look in BAM folder
+ if (fn_idx != NULL) {
+ idx = sam_index_load2(data[i]->fp, fn[i], fn_idx[i]);
+ } else {
+ idx = sam_index_load(data[i]->fp, fn[i]);
+ }
+
if (idx == NULL) {
fprintf(samtools_stderr, "[%s] fail to load index for %s\n", __func__, fn[i]);
exit(EXIT_FAILURE);
if (i == 0) h = data[i]->h = h_tmp; // save the header of the first file
else {
// FIXME: check consistency between h and h_tmp
- bam_hdr_destroy(h_tmp);
+ sam_hdr_destroy(h_tmp);
// we store only the first file's header; it's (alleged to be)
// compatible with the i-th file's target_name lookup needs
// Translate BAM @SQ tags to BCF ##contig tags
// todo: use/write new BAM header manipulation routines, fill also UR, M5
- for (i=0; i<h->n_targets; i++)
+ for (i=0; i < sam_hdr_nref(h); i++)
{
str.l = 0;
- ksprintf(&str, "##contig=<ID=%s,length=%d>", h->target_name[i], h->target_len[i]);
+ ksprintf(&str, "##contig=<ID=%s,length=%"PRId64">", sam_hdr_tid2name(h, i), (int64_t) sam_hdr_tid2len(h, i));
bcf_hdr_append(bcf_hdr, str.s);
}
free(str.s);
for (i=0; i<sm->n; i++)
bcf_hdr_add_sample(bcf_hdr, sm->smpl[i]);
bcf_hdr_add_sample(bcf_hdr, NULL);
- bcf_hdr_write(bcf_fp, bcf_hdr);
+ if (bcf_hdr_write(bcf_fp, bcf_hdr) != 0) {
+ print_error_errno("mpileup", "Failed to write VCF/BCF header to \"%s\"",
+ conf->output_fname? conf->output_fname : "standard output");
+ exit(EXIT_FAILURE);
+ }
// End of BCF header creation
// Initialise the calling algorithm
bam_mplp_set_maxcnt(iter, max_depth);
bcf1_t *bcf_rec = bcf_init1();
int ret;
- int last_tid = -1, last_pos = -1;
+ int last_tid = -1;
+ hts_pos_t last_pos = -1;
// begin pileup
- while ( (ret=bam_mplp_auto(iter, &tid, &pos, n_plp, plp)) > 0) {
+ while ( (ret=bam_mplp64_auto(iter, &tid, &pos, n_plp, plp)) > 0) {
if (conf->reg && (pos < beg0 || pos >= end0)) continue; // out of the region requested
mplp_get_ref(data[0], tid, &ref, &ref_len);
//printf("tid=%d len=%d ref=%p/%s\n", tid, ref_len, ref, ref);
if (conf->flag & MPLP_BCF) {
int total_depth, _ref0, ref16;
- if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, h->target_name[tid], pos, pos+1)) continue;
+ if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, sam_hdr_tid2name(h, tid), pos, pos+1)) continue;
for (i = total_depth = 0; i < n; ++i) total_depth += n_plp[i];
group_smpl(&gplp, sm, &buf, n, fn, n_plp, plp, conf->flag & MPLP_IGNORE_RG);
_ref0 = (ref && pos < ref_len)? ref[pos] : 'N';
bcf_call_combine(gplp.n, bcr, bca, ref16, &bc);
bcf_clear1(bcf_rec);
bcf_call2bcf(&bc, bcf_rec, bcr, conf->fmt_flag, 0, 0);
- bcf_write1(bcf_fp, bcf_hdr, bcf_rec);
+ if (bcf_write1(bcf_fp, bcf_hdr, bcf_rec) != 0) {
+ print_error_errno("mpileup", "Failed to write VCF/BCF record to \"%s\"",
+ conf->output_fname?conf->output_fname:"standard output");
+ exit(EXIT_FAILURE);
+ }
// call indels; todo: subsampling with total_depth>max_indel_depth instead of ignoring?
if (!(conf->flag&MPLP_NO_INDEL) && total_depth < max_indel_depth && bcf_call_gap_prep(gplp.n, gplp.n_plp, gplp.plp, pos, bca, ref, rghash) >= 0)
{
if (bcf_call_combine(gplp.n, bcr, bca, -1, &bc) >= 0) {
bcf_clear1(bcf_rec);
bcf_call2bcf(&bc, bcf_rec, bcr, conf->fmt_flag, bca, ref);
- bcf_write1(bcf_fp, bcf_hdr, bcf_rec);
+ if (bcf_write1(bcf_fp, bcf_hdr, bcf_rec) != 0) {
+ print_error_errno("mpileup", "Failed to write VCF/BCF record to \"%s\"",
+ conf->output_fname?conf->output_fname:"standard output");
+ exit(EXIT_FAILURE);
+ }
}
}
} else {
// Deal with missing portions of previous tids
while (tid > last_tid) {
if (last_tid >= 0 && !conf->reg) {
- while (++last_pos < h->target_len[last_tid]) {
- if (conf->bed && bed_overlap(conf->bed, h->target_name[last_tid], last_pos, last_pos + 1) == 0)
+ while (++last_pos < sam_hdr_tid2len(h, last_tid)) {
+ if (conf->bed && bed_overlap(conf->bed, sam_hdr_tid2name(h, last_tid), last_pos, last_pos + 1) == 0)
continue;
- print_empty_pileup(pileup_fp, conf, h->target_name[last_tid], last_pos, n, ref, ref_len);
+ print_empty_pileup(pileup_fp, conf, sam_hdr_tid2name(h, last_tid), last_pos, n, ref, ref_len);
}
}
last_tid++;
// Deal with missing portion of current tid
while (++last_pos < pos) {
if (conf->reg && last_pos < beg0) continue; // out of range; skip
- if (conf->bed && bed_overlap(conf->bed, h->target_name[tid], last_pos, last_pos + 1) == 0)
+ if (conf->bed && bed_overlap(conf->bed, sam_hdr_tid2name(h, tid), last_pos, last_pos + 1) == 0)
continue;
- print_empty_pileup(pileup_fp, conf, h->target_name[tid], last_pos, n, ref, ref_len);
+ print_empty_pileup(pileup_fp, conf, sam_hdr_tid2name(h, tid), last_pos, n, ref, ref_len);
}
last_tid = tid;
last_pos = pos;
}
- if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, h->target_name[tid], pos, pos+1)) continue;
+ if (conf->bed && tid >= 0 && !bed_overlap(conf->bed, sam_hdr_tid2name(h, tid), pos, pos+1)) continue;
- fprintf(pileup_fp, "%s\t%d\t%c", h->target_name[tid], pos + 1, (ref && pos < ref_len)? ref[pos] : 'N');
+ fprintf(pileup_fp, "%s\t%"PRIhts_pos"\t%c", sam_hdr_tid2name(h, tid), pos + 1, (ref && pos < ref_len)? ref[pos] : 'N');
for (i = 0; i < n; ++i) {
int j, cnt;
for (j = cnt = 0; j < n_plp[i]; ++j) {
fprintf(pileup_fp, "\t%d\t", cnt);
if (n_plp[i] == 0) {
fputs("*\t*", pileup_fp);
- if (conf->flag & MPLP_PRINT_MAPQ) fputs("\t*", pileup_fp);
- if (conf->flag & MPLP_PRINT_POS) fputs("\t*", pileup_fp);
- if (conf->flag & MPLP_PRINT_QNAME) fputs("\t*", pileup_fp);
+ if (conf->flag & MPLP_PRINT_QPOS)
+ fputs("\t*", pileup_fp);
+ int flag_value = MPLP_PRINT_QNAME;
+ while(flag_value < MPLP_PRINT_QUAL + 1) {
+ if (conf->flag & flag_value)
+ fputs("\t*", pileup_fp);
+ flag_value <<= 1;
+ }
+ if (conf->auxlist) {
+ int t = 0;
+ while(t++ < ((klist_t(auxlist) *)conf->auxlist)->size)
+ fputs("\t*", pileup_fp);
+ }
} else {
int n = 0;
+ kstring_t ks = KS_INITIALIZE;
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
int c = p->qpos < p->b->core.l_qseq
? bam_get_qual(p->b)[p->qpos]
: 0;
- if (c >= conf->min_baseQ)
- n++, pileup_seq(pileup_fp, plp[i] + j, pos, ref_len, ref);
+ if (c >= conf->min_baseQ) {
+ n++;
+ if (pileup_seq(pileup_fp, plp[i] + j, pos, ref_len, ref, &ks, conf->rev_del) < 0) {
+ ret = 1;
+ goto fail;
+ }
+ }
}
if (!n) putc('*', pileup_fp);
+ /* Print base qualities */
n = 0;
+ ks_free(&ks);
putc('\t', pileup_fp);
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
}
if (!n) putc('*', pileup_fp);
- if (conf->flag & MPLP_PRINT_MAPQ) {
+ /* Print mpileup positions */
+ if (conf->flag & MPLP_PRINT_QPOS) {
n = 0;
putc('\t', pileup_fp);
for (j = 0; j < n_plp[i]; ++j) {
const bam_pileup1_t *p = plp[i] + j;
int c = p->qpos < p->b->core.l_qseq
- ? bam_get_qual(p->b)[p->qpos]
- : 0;
+ ? bam_get_qual(p->b)[p->qpos]
+ : 0;
if ( c < conf->min_baseQ ) continue;
- c = plp[i][j].b->core.qual + 33;
- if (c > 126) c = 126;
- putc(c, pileup_fp);
+ if (n > 0) putc(',', pileup_fp);
n++;
+ fprintf(pileup_fp, "%d", p->qpos + 1);
}
if (!n) putc('*', pileup_fp);
}
- if (conf->flag & MPLP_PRINT_POS) {
- n = 0;
- putc('\t', pileup_fp);
- for (j = 0; j < n_plp[i]; ++j) {
- const bam_pileup1_t *p = plp[i] + j;
- int c = p->qpos < p->b->core.l_qseq
- ? bam_get_qual(p->b)[p->qpos]
- : 0;
- if ( c < conf->min_baseQ ) continue;
-
- if (n > 0) putc(',', pileup_fp);
- fprintf(pileup_fp, "%d", plp[i][j].qpos + 1); // FIXME: fprintf(samtools_stdout, ) is very slow...
- n++;
+ /* Print selected columns */
+ int flag_value = MPLP_PRINT_QNAME;
+ while(flag_value < MPLP_PRINT_QUAL + 1) {
+ if (conf->flag & flag_value) {
+ n = 0;
+ putc('\t', pileup_fp);
+ for (j = 0; j < n_plp[i]; ++j) {
+ const bam_pileup1_t *p = &plp[i][j];
+ int c = p->qpos < p->b->core.l_qseq
+ ? bam_get_qual(p->b)[p->qpos]
+ : 0;
+ if ( c < conf->min_baseQ ) continue;
+ if (n > 0 && flag_value != MPLP_PRINT_MAPQ) putc(',', pileup_fp);
+ n++;
+
+ switch (flag_value) {
+ case MPLP_PRINT_QNAME:
+ fputs(bam_get_qname(p->b), pileup_fp);
+ break;
+ case MPLP_PRINT_FLAG:
+ fprintf(pileup_fp, "%d", p->b->core.flag);
+ break;
+ case MPLP_PRINT_RNAME:
+ if (p->b->core.tid >= 0)
+ fputs(sam_hdr_tid2name(h, p->b->core.tid), pileup_fp);
+ else
+ putc('*', pileup_fp);
+ break;
+ case MPLP_PRINT_POS:
+ fprintf(pileup_fp, "%"PRId64, (int64_t) p->b->core.pos + 1);
+ break;
+ case MPLP_PRINT_MAPQ:
+ c = p->b->core.qual + 33;
+ if (c > 126) c = 126;
+ putc(c, pileup_fp);
+ break;
+ case MPLP_PRINT_RNEXT:
+ if (p->b->core.mtid >= 0)
+ fputs(sam_hdr_tid2name(h, p->b->core.mtid), pileup_fp);
+ else
+ putc('*', pileup_fp);
+ break;
+ case MPLP_PRINT_PNEXT:
+ fprintf(pileup_fp, "%"PRId64, (int64_t) p->b->core.mpos + 1);
+ break;
+ }
+ }
+ if (!n) putc('*', pileup_fp);
}
- if (!n) putc('*', pileup_fp);
+ flag_value <<= 1;
}
- if (conf->flag & MPLP_PRINT_QNAME) {
- n = 0;
- putc('\t', pileup_fp);
- for (j = 0; j < n_plp[i]; ++j) {
- const bam_pileup1_t *p = &plp[i][j];
- int c = p->qpos < p->b->core.l_qseq
- ? bam_get_qual(p->b)[p->qpos]
- : 0;
- if ( c < conf->min_baseQ ) continue;
-
- if (n > 0) putc(',', pileup_fp);
- fputs(bam_get_qname(p->b), pileup_fp);
- n++;
+ /* Print selected tags */
+ klist_t(auxlist) *auxlist_p = ((klist_t(auxlist) *)conf->auxlist);
+ if (auxlist_p && auxlist_p->size) {
+ kliter_t(auxlist) *aux;
+ for (aux = kl_begin(auxlist_p); aux != kl_end(auxlist_p); aux = kl_next(aux)) {
+ n = 0;
+ putc('\t', pileup_fp);
+ for (j = 0; j < n_plp[i]; ++j) {
+ const bam_pileup1_t *p = &plp[i][j];
+ int c = p->qpos < p->b->core.l_qseq
+ ? bam_get_qual(p->b)[p->qpos]
+ : 0;
+ if ( c < conf->min_baseQ ) continue;
+
+ if (n > 0) putc(conf->sep, pileup_fp);
+ n++;
+ uint8_t* tag_u = bam_aux_get(p->b, kl_val(aux));
+ if (!tag_u) {
+ putc(conf->empty , pileup_fp);
+ continue;
+ }
+
+ /* Tag value is string */
+ if (*tag_u == 'Z' || *tag_u == 'H') {
+ char *tag_s = bam_aux2Z(tag_u);
+ if (!tag_s) continue;
+ fputs(tag_s, pileup_fp);
+ }
+
+ /* Tag value is integer */
+ if (*tag_u == 'I' || *tag_u == 'i' || *tag_u == 'C' || *tag_u == 'c' || *tag_u == 'S' || *tag_u == 's') {
+ int64_t tag_i = bam_aux2i(tag_u);
+ fprintf(pileup_fp, "%" PRId64 "", tag_i);
+ }
+
+ /* Tag value is float */
+ if (*tag_u == 'd' || *tag_u == 'f') {
+ double tag_f = bam_aux2f(tag_u);
+ fprintf(pileup_fp, "%lf", tag_f);
+ }
+
+ /* Tag value is character */
+ if (*tag_u == 'A') {
+ char tag_c = bam_aux2A(tag_u);
+ putc(tag_c, pileup_fp);
+ }
+ }
+ if (!n) putc('*', pileup_fp);
}
- if (!n) putc('*', pileup_fp);
}
}
}
last_pos = beg0-1;
mplp_get_ref(data[0], tid0, &ref, &ref_len);
}
- while (last_tid >= 0 && last_tid < h->n_targets) {
- while (++last_pos < h->target_len[last_tid]) {
+ while (last_tid >= 0 && last_tid < sam_hdr_nref(h)) {
+ while (++last_pos < sam_hdr_tid2len(h, last_tid)) {
if (last_pos >= end0) break;
- if (conf->bed && bed_overlap(conf->bed, h->target_name[last_tid], last_pos, last_pos + 1) == 0)
+ if (conf->bed && bed_overlap(conf->bed, sam_hdr_tid2name(h, last_tid), last_pos, last_pos + 1) == 0)
continue;
- print_empty_pileup(pileup_fp, conf, h->target_name[last_tid], last_pos, n, ref, ref_len);
+ print_empty_pileup(pileup_fp, conf, sam_hdr_tid2name(h, last_tid), last_pos, n, ref, ref_len);
}
last_tid++;
last_pos = -1;
}
}
+fail:
// clean up
free(bc.tmp.s);
bcf_destroy1(bcf_rec);
free(gplp.plp); free(gplp.n_plp); free(gplp.m_plp);
bcf_call_del_rghash(rghash);
bam_mplp_destroy(iter);
- bam_hdr_destroy(h);
+ sam_hdr_destroy(h);
for (i = 0; i < n; ++i) {
sam_close(data[i]->fp);
if (data[i]->iter) hts_itr_destroy(data[i]->iter);
" [%s]\n", tmp_filter);
fprintf(fp,
" -x, --ignore-overlaps disable read-pair overlap detection\n"
+" -X, --customized-index use customized index files\n" // -X flag for index filename
"\n"
"Output options:\n"
" -o, --output FILE write output to FILE [standard output]\n"
" -O, --output-BP output base positions on reads\n"
" -s, --output-MQ output mapping quality\n"
" --output-QNAME output read names\n"
+" --output-extra STR output extra read fields and read tag values\n"
+" --output-sep CHAR set the separator character for tag lists [,]\n"
+" --output-empty CHAR set the no value character for tag lists [*]\n"
+" --reverse-del use '#' character for deletions on the reverse strand\n"
" -a output all positions (including zero depth)\n"
" -a -a (or -aa) output absolutely all positions, including unused ref. sequences\n"
"\n"
"Generic options:\n");
- sam_global_opt_help(fp, "-.--.-");
+ sam_global_opt_help(fp, "-.--.--.");
fprintf(fp, "\n"
"Note that using \"samtools mpileup\" to generate BCF or VCF files is now\n"
int c;
const char *file_list = NULL;
char **fn = NULL;
- int nfiles = 0, use_orphan = 0;
+ int nfiles = 0, use_orphan = 0, has_index_file = 0;
mplp_conf_t mplp;
memset(&mplp, 0, sizeof(mplp_conf_t));
mplp.min_baseQ = 13;
mplp.rflag_filter = BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP;
mplp.output_fname = NULL;
mplp.all = 0;
+ mplp.rev_del = 0;
+ mplp.sep = ',';
+ mplp.empty = '*';
sam_global_args_init(&mplp.ga);
static const struct option lopts[] =
{"per-sample-mF", no_argument, NULL, 'p'},
{"per-sample-mf", no_argument, NULL, 'p'},
{"platforms", required_argument, NULL, 'P'},
+ {"customized-index", no_argument, NULL, 'X'},
+ {"reverse-del", no_argument, NULL, 6},
+ {"output-extra", required_argument, NULL, 7},
+ {"output-sep", required_argument, NULL, 8},
+ {"output-empty", required_argument, NULL, 9},
{NULL, 0, NULL, 0}
};
- while ((c = getopt_long(argc, argv, "Agf:r:l:q:Q:uRC:BDSd:L:b:P:po:e:h:Im:F:EG:6OsVvxt:a",lopts,NULL)) >= 0) {
+
+ while ((c = getopt_long(argc, argv, "Agf:r:l:q:Q:uRC:BDSd:L:b:P:po:e:h:Im:F:EG:6OsVvxXt:a",lopts,NULL)) >= 0) {
switch (c) {
case 'x': mplp.flag &= ~MPLP_SMART_OVERLAPS; break;
case 1 :
case 3 : mplp.output_fname = optarg; break;
case 4 : mplp.openQ = atoi(optarg); break;
case 5 : mplp.flag |= MPLP_PRINT_QNAME; break;
+ case 6 : mplp.rev_del = 1; break;
+ case 7 :
+ if (build_auxlist(&mplp, optarg) != 0) {
+ fprintf(samtools_stderr,"Could not build aux list using '%s'\n", optarg);
+ return 1;
+ }
+ break;
+ case 8: mplp.sep = optarg[0]; break;
+ case 9: mplp.empty = optarg[0]; break;
case 'f':
mplp.fai = fai_load(optarg);
if (mplp.fai == NULL) return 1;
case 'v': mplp.flag |= MPLP_BCF | MPLP_VCF; deprecated(c); break;
case 'u': mplp.flag |= MPLP_NO_COMP | MPLP_BCF; deprecated(c); break;
case 'B': mplp.flag &= ~MPLP_REALN; break;
+ case 'X': has_index_file = 1; break;
case 'D': mplp.fmt_flag |= B2B_FMT_DP; deprecated(c); break;
case 'S': mplp.fmt_flag |= B2B_FMT_SP; deprecated(c); break;
case 'V': mplp.fmt_flag |= B2B_FMT_DV; deprecated(c); break;
case '6': mplp.flag |= MPLP_ILLUMINA13; break;
case 'R': mplp.flag |= MPLP_IGNORE_RG; break;
case 's': mplp.flag |= MPLP_PRINT_MAPQ; break;
- case 'O': mplp.flag |= MPLP_PRINT_POS; break;
+ case 'O': mplp.flag |= MPLP_PRINT_QPOS; break;
case 'C': mplp.capQ_thres = atoi(optarg); break;
case 'q': mplp.min_mq = atoi(optarg); break;
case 'Q': mplp.min_baseQ = atoi(optarg); break;
}
int ret;
if (file_list) {
+ if (has_index_file) {
+ fprintf(samtools_stderr,"Error: The -b option cannot be combined with -X\n"); // No customize index loc in file list mode
+ return 1;
+ }
if ( read_file_list(file_list,&nfiles,&fn) ) return 1;
- ret = mpileup(&mplp,nfiles,fn);
+ ret = mpileup(&mplp,nfiles,fn,NULL);
for (c=0; c<nfiles; c++) free(fn[c]);
free(fn);
}
- else
- ret = mpileup(&mplp, argc - optind, argv + optind);
+ else {
+ if (has_index_file) {
+ if ((argc - optind)%2 !=0) { // Calculate # of input BAM files
+ fprintf(samtools_stderr, "Odd number of filenames detected! Each BAM file should have an index file\n");
+ return 1;
+ }
+ nfiles = (argc - optind)/2;
+ ret = mpileup(&mplp, nfiles, argv + optind, argv + nfiles + optind);
+ } else {
+ nfiles = argc - optind;
+ ret = mpileup(&mplp, nfiles, argv + optind, NULL);
+ }
+ }
if (mplp.rghash) khash_str2int_destroy_free(mplp.rghash);
free(mplp.reg); free(mplp.pl_list);
if (mplp.fai) fai_destroy(mplp.fai);
if (mplp.bed) bed_destroy(mplp.bed);
+ if (mplp.auxlist) kl_destroy(auxlist, (klist_t(auxlist) *)mplp.auxlist);
return ret;
}
/* bam_quickcheck.c -- quickcheck subcommand.
- Copyright (C) 2015 Genome Research Ltd.
+ Copyright (C) 2015-2017 Genome Research Ltd.
Author: Joshua C. Randall <jcrandall@alum.mit.edu>
"Options:\n"
" -v verbose output (repeat for more verbosity)\n"
" -q suppress warning messages\n"
+" -u unmapped input (do not require targets in header)\n"
"\n"
"Notes:\n"
"\n"
int main_quickcheck(int argc, char** argv)
{
- int verbose = 0, quiet = 0;
+ int verbose = 0, quiet = 0, unmapped = 0;
hts_verbose = 0;
- const char* optstring = "vq";
+ const char* optstring = "vqu";
int opt;
while ((opt = getopt(argc, argv, optstring)) != -1) {
switch (opt) {
+ case 'u':
+ unmapped = 1;
+ break;
case 'v':
verbose++;
break;
else {
if (verbose >= 3) fprintf(stderr, "%s is sequence data\n", fn);
// check header
- bam_hdr_t *header = sam_hdr_read(hts_fp);
+ sam_hdr_t *header = sam_hdr_read(hts_fp);
if (header == NULL) {
QC_ERR(QC_BAD_HEADER, 2, "%s caused an error whilst reading its header.\n", fn);
} else {
- if (header->n_targets <= 0) {
+ if (!unmapped && sam_hdr_nref(header) <= 0) {
QC_ERR(QC_BAD_HEADER, 2, "%s had no targets in header.\n", fn);
}
else {
- if (verbose >= 3) fprintf(stderr, "%s has %d targets in header.\n", fn, header->n_targets);
+ if (verbose >= 3) fprintf(stderr, "%s has %d targets in header.\n", fn, sam_hdr_nref(header));
}
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
}
}
// check EOF on formats that support this
/* bam_quickcheck.c -- quickcheck subcommand.
- Copyright (C) 2015 Genome Research Ltd.
+ Copyright (C) 2015-2017 Genome Research Ltd.
Author: Joshua C. Randall <jcrandall@alum.mit.edu>
"Options:\n"
" -v verbose output (repeat for more verbosity)\n"
" -q suppress warning messages\n"
+" -u unmapped input (do not require targets in header)\n"
"\n"
"Notes:\n"
"\n"
int main_quickcheck(int argc, char** argv)
{
- int verbose = 0, quiet = 0;
+ int verbose = 0, quiet = 0, unmapped = 0;
hts_verbose = 0;
- const char* optstring = "vq";
+ const char* optstring = "vqu";
int opt;
while ((opt = getopt(argc, argv, optstring)) != -1) {
switch (opt) {
+ case 'u':
+ unmapped = 1;
+ break;
case 'v':
verbose++;
break;
else {
if (verbose >= 3) fprintf(samtools_stderr, "%s is sequence data\n", fn);
// check header
- bam_hdr_t *header = sam_hdr_read(hts_fp);
+ sam_hdr_t *header = sam_hdr_read(hts_fp);
if (header == NULL) {
QC_ERR(QC_BAD_HEADER, 2, "%s caused an error whilst reading its header.\n", fn);
} else {
- if (header->n_targets <= 0) {
+ if (!unmapped && sam_hdr_nref(header) <= 0) {
QC_ERR(QC_BAD_HEADER, 2, "%s had no targets in header.\n", fn);
}
else {
- if (verbose >= 3) fprintf(samtools_stderr, "%s has %d targets in header.\n", fn, header->n_targets);
+ if (verbose >= 3) fprintf(samtools_stderr, "%s has %d targets in header.\n", fn, sam_hdr_nref(header));
}
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
}
}
// check EOF on formats that support this
/* bam_reheader.c -- reheader subcommand.
Copyright (C) 2010 Broad Institute.
- Copyright (C) 2012-2015 Genome Research Ltd.
+ Copyright (C) 2012-2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
#include <stdlib.h>
#include <assert.h>
#include <getopt.h>
+#include <unistd.h>
#include "htslib/bgzf.h"
#include "htslib/sam.h"
* Reads a file and outputs a new BAM file to fd with 'h' replaced as
* the header. No checks are made to the validity.
*/
-int bam_reheader(BGZF *in, bam_hdr_t *h, int fd,
- const char *arg_list, int add_PG)
+int bam_reheader(BGZF *in, sam_hdr_t *h, int fd,
+ const char *arg_list, int no_pg, int skip_header)
{
BGZF *fp = NULL;
ssize_t len;
uint8_t *buf = NULL;
- SAM_hdr *sh = NULL;
+ sam_hdr_t *tmp;
+ if (!h)
+ return -1;
+
if (in->is_write) return -1;
buf = malloc(BUF_SIZE);
if (!buf) {
fprintf(stderr, "Out of memory\n");
return -1;
}
- if (bam_hdr_read(in) == NULL) {
- fprintf(stderr, "Couldn't read header\n");
- goto fail;
+
+ if (!skip_header) {
+ if ((tmp = bam_hdr_read(in)) == NULL) {
+ fprintf(stderr, "Couldn't read header\n");
+ goto fail;
+ }
+ sam_hdr_destroy(tmp);
}
+
fp = bgzf_fdopen(fd, "w");
if (!fp) {
print_error_errno("reheader", "Couldn't open output file");
goto fail;
}
- if (add_PG) {
- // Around the houses, but it'll do until we can manipulate bam_hdr_t natively.
- sh = sam_hdr_parse_(h->text, h->l_text);
- if (!sh)
- goto fail;
- if (sam_hdr_add_PG(sh, "samtools",
+ if (!no_pg && sam_hdr_add_pg(h, "samtools",
"VN", samtools_version(),
arg_list ? "CL": NULL,
arg_list ? arg_list : NULL,
NULL) != 0)
goto fail;
- free(h->text);
- h->text = strdup(sam_hdr_str(sh));
- h->l_text = sam_hdr_length(sh);
- if (!h->text)
- goto fail;
- sam_hdr_free(sh);
- sh = NULL;
- }
-
if (bam_hdr_write(fp, h) < 0) {
print_error_errno("reheader", "Couldn't write header");
goto fail;
fail:
bgzf_close(fp);
free(buf);
- sam_hdr_free(sh);
return -1;
}
*
* FIXME: error checking
*/
-int cram_reheader(cram_fd *in, bam_hdr_t *h, const char *arg_list, int add_PG)
+int cram_reheader(cram_fd *in, sam_hdr_t *h, const char *arg_list, int no_pg)
{
htsFile *h_out = hts_open("-", "wc");
cram_fd *out = h_out->fp.cram;
cram_container *c = NULL;
int ret = -1;
+ if (!h)
+ return ret;
// Attempt to fill out a cram->refs[] array from @SQ headers
- cram_fd_set_header(out, sam_hdr_parse_(h->text, h->l_text));
- if (add_PG) {
- if (sam_hdr_add_PG(cram_fd_get_header(out), "samtools",
+ sam_hdr_t *cram_h = sam_hdr_dup(h);
+ if (!cram_h)
+ return -1;
+ cram_fd_set_header(out, cram_h);
+ if (!no_pg && sam_hdr_add_pg(cram_fd_get_header(out), "samtools",
"VN", samtools_version(),
arg_list ? "CL": NULL,
arg_list ? arg_list : NULL,
- NULL) != 0)
+ NULL))
goto err;
- // Covert back to bam_hdr_t struct
- free(h->text);
- h->text = strdup(sam_hdr_str(cram_fd_get_header(out)));
- h->l_text = sam_hdr_length(cram_fd_get_header(out));
- if (!h->text)
- goto err;
- }
-
- if (sam_hdr_write(h_out, h) != 0)
+ if (sam_hdr_write(h_out, cram_h) != 0)
goto err;
cram_set_option(out, CRAM_OPT_REFERENCE, NULL);
* -1 on general failure;
* -2 on failure due to insufficient size
*/
-int cram_reheader_inplace2(cram_fd *fd, const bam_hdr_t *h, const char *arg_list,
- int add_PG)
+int cram_reheader_inplace2(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
+ int no_pg)
{
cram_container *c = NULL;
cram_block *b = NULL;
- SAM_hdr *hdr = NULL;
+ sam_hdr_t *cram_h = NULL;
off_t start;
int ret = -1;
+ if (!h)
+ goto err;
if (cram_major_vers(fd) < 2 ||
cram_major_vers(fd) > 3) {
goto err;
}
- if (!(hdr = sam_hdr_parse_(h->text, h->l_text)))
+ cram_h = sam_hdr_dup(h);
+ if (!cram_h)
goto err;
- if (add_PG && sam_hdr_add_PG(hdr, "samtools", "VN", samtools_version(),
+ if (!no_pg && sam_hdr_add_pg(cram_h, "samtools", "VN", samtools_version(),
arg_list ? "CL": NULL,
arg_list ? arg_list : NULL,
NULL))
goto err;
- int header_len = sam_hdr_length(hdr);
+ int header_len = sam_hdr_length(cram_h);
/* Fix M5 strings? Maybe out of scope for this tool */
// Load the existing header
cram_block_set_offset(b, 0); // rewind block
int32_put_blk(b, header_len);
- cram_block_append(b, sam_hdr_str(hdr), header_len);
+ cram_block_append(b, (void *)sam_hdr_str(cram_h), header_len);
// Zero the remaining block
memset((char *)cram_block_get_data(b)+cram_block_get_offset(b), 0,
cram_block_get_uncomp_size(b) - cram_block_get_offset(b));
err:
if (c) cram_free_container(c);
if (b) cram_free_block(b);
- if (hdr) sam_hdr_free(hdr);
+ if (cram_h) sam_hdr_destroy(cram_h);
return ret;
}
* -1 on general failure;
* -2 on failure due to insufficient size
*/
-int cram_reheader_inplace3(cram_fd *fd, const bam_hdr_t *h, const char *arg_list,
- int add_PG)
+int cram_reheader_inplace3(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
+ int no_pg)
{
cram_container *c = NULL;
cram_block *b = NULL;
- SAM_hdr *hdr = NULL;
+ sam_hdr_t *cram_h = NULL;
off_t start, sz, end;
int container_sz, max_container_sz;
char *buf = NULL;
int ret = -1;
+ if (!h)
+ goto err;
if (cram_major_vers(fd) < 2 ||
cram_major_vers(fd) > 3) {
goto err;
}
- if (!(hdr = sam_hdr_parse_(h->text, h->l_text)))
+ cram_h = sam_hdr_dup(h);
+ if (!cram_h)
goto err;
- if (add_PG && sam_hdr_add_PG(hdr, "samtools", "VN", samtools_version(),
+ if (!no_pg && sam_hdr_add_pg(cram_h, "samtools", "VN", samtools_version(),
arg_list ? "CL": NULL,
arg_list ? arg_list : NULL,
NULL))
goto err;
- int header_len = sam_hdr_length(hdr);
+ int header_len = sam_hdr_length(cram_h);
/* Fix M5 strings? Maybe out of scope for this tool */
// Find current size of SAM header block
// Version 3.0 supports compressed header
b = cram_new_block(FILE_HEADER, 0);
int32_put_blk(b, header_len);
- cram_block_append(b, sam_hdr_str(hdr), header_len);
+ cram_block_append(b, (void *)sam_hdr_str(cram_h), header_len);
cram_block_update_size(b);
cram_compress_block(fd, b, NULL, -1, -1);
if (c) cram_free_container(c);
if (buf) free(buf);
if (b) cram_free_block(b);
- if (hdr) sam_hdr_free(hdr);
+ if (cram_h) sam_hdr_destroy(cram_h);
return ret;
}
-int cram_reheader_inplace(cram_fd *fd, const bam_hdr_t *h, const char *arg_list,
- int add_PG)
+int cram_reheader_inplace(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
+ int no_pg)
{
switch (cram_major_vers(fd)) {
- case 2: return cram_reheader_inplace2(fd, h, arg_list, add_PG);
- case 3: return cram_reheader_inplace3(fd, h, arg_list, add_PG);
+ case 2: return cram_reheader_inplace2(fd, h, arg_list, no_pg);
+ case 3: return cram_reheader_inplace3(fd, h, arg_list, no_pg);
default:
fprintf(stderr, "[%s] unsupported CRAM version %d\n", __func__,
cram_major_vers(fd));
static void usage(FILE *fp, int ret) {
fprintf(fp,
"Usage: samtools reheader [-P] in.header.sam in.bam > out.bam\n"
- " or samtools reheader [-P] -i in.header.sam file.bam\n"
+ " or samtools reheader [-P] -i in.header.sam file.cram\n"
+ " or samtools reheader -c CMD in.bam\n"
+ " or samtools reheader -c CMD in.cram\n"
"\n"
"Options:\n"
- " -P, --no-PG Do not generate an @PG header line.\n"
- " -i, --in-place Modify the bam/cram file directly.\n"
- " (Defaults to outputting to stdout.)\n");
+ " -P, --no-PG Do not generate a @PG header line.\n"
+ " -i, --in-place Modify the CRAM file directly, if possible.\n"
+ " (Defaults to outputting to stdout.)\n"
+ " -c, --command CMD Pass the header in SAM format to external program CMD.\n");
exit(ret);
}
+static sam_hdr_t* external_reheader(samFile* in, const char* external) {
+ char *command = NULL;
+ sam_hdr_t* h = NULL;
+ sam_hdr_t* ih = sam_hdr_read(in);
+ if (ih == NULL) {
+ fprintf(stderr, "[%s] failed to read the header for '%s'.\n", __func__, in->fn);
+ return NULL;
+ }
+ char tmp_fn[] = "reheaderXXXXXX";
+ int tmp_fd = mkstemp(tmp_fn);
+ if (tmp_fd < 0) {
+ print_error_errno("reheader", "fail to open temp file '%s'", tmp_fn);
+ return NULL;
+ }
+ hFILE* tmp_hf = hdopen(tmp_fd, "w");
+ if (!tmp_hf) {
+ fprintf(stderr, "[%s] failed to convert to hFILE.\n", __func__);
+ goto cleanup;
+ }
+ samFile* tmp_sf = hts_hopen(tmp_hf, tmp_fn, "w");
+ if (!tmp_sf) {
+ fprintf(stderr, "[%s] failed to convert to samFile.\n", __func__);
+ goto cleanup;
+ }
+ if (-1 == sam_hdr_write(tmp_sf, ih)) {
+ fprintf(stderr, "[%s] failed to write the header to the temp file.\n", __func__);
+ goto cleanup;
+ }
+ sam_close(tmp_sf);
+ sam_hdr_destroy(ih);
+ int comm_len = strlen(external) + strlen(tmp_fn) + 8;
+ command = calloc(comm_len, 1);
+ if (!command || snprintf(command, comm_len, "( %s ) < %s", external, tmp_fn) != comm_len - 1) {
+ fprintf(stderr, "[%s] failed to create command string.\n", __func__);
+ goto cleanup;
+ }
+ FILE* nh = popen(command, "r");
+ if (!nh) {
+ print_error_errno("reheader", "[%s] failed to run external command '%s'.\n", __func__, command);
+ goto cleanup;
+ }
+
+ int nh_fd = dup(fileno(nh));
+ if (nh_fd < 0) {
+ fprintf(stderr, "[%s] failed to get the file descriptor.\n", __func__);
+ goto cleanup;
+ }
+ hFILE* nh_hf = hdopen(nh_fd, "r");
+ if (!nh_hf) {
+ fprintf(stderr, "[%s] failed to convert to hFILE.\n", __func__);
+ goto cleanup;
+ }
+ samFile* nh_sf = hts_hopen(nh_hf, tmp_fn, "r");
+ if (!nh_sf) {
+ fprintf(stderr, "[%s] failed to convert to samFile.\n", __func__);
+ goto cleanup;
+ }
+
+ h = sam_hdr_read(nh_sf);
+ sam_close(nh_sf);
+ if (h == NULL) {
+ fprintf(stderr, "[%s] failed to read the header from the temp file.\n", __func__);
+ }
+ int res = pclose(nh);
+ if (res != 0) {
+ if (res < 0) {
+ print_error_errno("reheader",
+ "Error on closing pipe from command '%s'.\n",
+ command);
+ } else {
+ print_error("reheader",
+ "Non-zero exit code returned by command '%s'\n",
+ command);
+ }
+ if (h) sam_hdr_destroy(h);
+ h = NULL;
+ }
+cleanup:
+ free(command);
+ if (unlink(tmp_fn) != 0) {
+ print_error_errno("reheader", "failed to remove the temp file '%s'", tmp_fn);
+ }
+
+ return h;
+}
+
int main_reheader(int argc, char *argv[])
{
- int inplace = 0, r, add_PG = 1, c;
- bam_hdr_t *h;
+ int inplace = 0, r, no_pg = 0, c, skip_header = 0;
+ sam_hdr_t *h;
samFile *in;
- char *arg_list = stringify_argv(argc+1, argv-1);
+ char *arg_list = NULL, *external = NULL;
static const struct option lopts[] = {
{"help", no_argument, NULL, 'h'},
{"in-place", no_argument, NULL, 'i'},
{"no-PG", no_argument, NULL, 'P'},
+ {"command", required_argument, NULL, 'c'},
{NULL, 0, NULL, 0}
};
- while ((c = getopt_long(argc, argv, "hiP", lopts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "hiPc:", lopts, NULL)) >= 0) {
switch (c) {
- case 'P': add_PG = 0; break;
+ case 'P': no_pg = 1; break;
case 'i': inplace = 1; break;
+ case 'c': external = optarg; break;
case 'h': usage(stdout, 0); break;
default:
fprintf(stderr, "Invalid option '%c'\n", c);
}
}
- if (argc - optind != 2)
+ if ((argc - optind != 2 || external) && (argc - optind != 1 || !external))
usage(stderr, 1);
- { // read the header
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("reheader", "failed to create arg_list");
+ return 1;
+ }
+
+ if (external) {
+ skip_header = 1;
+ in = sam_open(argv[optind], inplace?"r+":"r");
+ if (in == 0) {
+ print_error_errno("reheader", "fail to open file '%s'", argv[optind]);
+ return 1;
+ }
+
+ h = external_reheader(in, external);
+ if (h == NULL) {
+ fprintf(stderr, "[%s] failed to read the header from '%s'.\n", __func__, external);
+ sam_close(in);
+ return 1;
+ }
+ } else { // read the header from a separate file
samFile *fph = sam_open(argv[optind], "r");
if (fph == 0) {
print_error_errno("reheader", "fail to read the header from '%s'", argv[optind]);
__func__, argv[1]);
return 1;
}
+ in = sam_open(argv[optind+1], inplace?"r+":"r");
+ if (in == 0) {
+ print_error_errno("reheader", "fail to open file '%s'", argv[optind+1]);
+ return 1;
+ }
}
- in = sam_open(argv[optind+1], inplace?"r+":"r");
- if (in == 0) {
- print_error_errno("reheader", "fail to open file '%s'", argv[optind+1]);
- return 1;
- }
+
if (hts_get_format(in)->format == bam) {
- r = bam_reheader(in->fp.bgzf, h, fileno(stdout), arg_list, add_PG);
- } else {
+ if (inplace) {
+ print_error("reheader", "cannot reheader BAM '%s' in-place", argv[optind+1]);
+ r = -1;
+ } else {
+ r = bam_reheader(in->fp.bgzf, h, fileno(stdout), arg_list, no_pg, skip_header);
+ }
+ } else if (hts_get_format(in)->format == cram) {
if (inplace)
- r = cram_reheader_inplace(in->fp.cram, h, arg_list, add_PG);
+ r = cram_reheader_inplace(in->fp.cram, h, arg_list, no_pg);
else
- r = cram_reheader(in->fp.cram, h, arg_list, add_PG);
+ r = cram_reheader(in->fp.cram, h, arg_list, no_pg);
+ } else {
+ print_error("reheader", "input file '%s' must be BAM or CRAM", argv[optind+1]);
+ r = -1;
}
if (sam_close(in) != 0)
r = -1;
- bam_hdr_destroy(h);
+ sam_hdr_destroy(h);
if (arg_list)
free(arg_list);
/* bam_reheader.c -- reheader subcommand.
Copyright (C) 2010 Broad Institute.
- Copyright (C) 2012-2015 Genome Research Ltd.
+ Copyright (C) 2012-2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
#include <stdlib.h>
#include <assert.h>
#include <getopt.h>
+#include <unistd.h>
#include "htslib/bgzf.h"
#include "htslib/sam.h"
* Reads a file and outputs a new BAM file to fd with 'h' replaced as
* the header. No checks are made to the validity.
*/
-int bam_reheader(BGZF *in, bam_hdr_t *h, int fd,
- const char *arg_list, int add_PG)
+int bam_reheader(BGZF *in, sam_hdr_t *h, int fd,
+ const char *arg_list, int no_pg, int skip_header)
{
BGZF *fp = NULL;
ssize_t len;
uint8_t *buf = NULL;
- SAM_hdr *sh = NULL;
+ sam_hdr_t *tmp;
+ if (!h)
+ return -1;
+
if (in->is_write) return -1;
buf = malloc(BUF_SIZE);
if (!buf) {
fprintf(samtools_stderr, "Out of memory\n");
return -1;
}
- if (bam_hdr_read(in) == NULL) {
- fprintf(samtools_stderr, "Couldn't read header\n");
- goto fail;
+
+ if (!skip_header) {
+ if ((tmp = bam_hdr_read(in)) == NULL) {
+ fprintf(samtools_stderr, "Couldn't read header\n");
+ goto fail;
+ }
+ sam_hdr_destroy(tmp);
}
+
fp = bgzf_fdopen(fd, "w");
if (!fp) {
print_error_errno("reheader", "Couldn't open output file");
goto fail;
}
- if (add_PG) {
- // Around the houses, but it'll do until we can manipulate bam_hdr_t natively.
- sh = sam_hdr_parse_(h->text, h->l_text);
- if (!sh)
- goto fail;
- if (sam_hdr_add_PG(sh, "samtools",
+ if (!no_pg && sam_hdr_add_pg(h, "samtools",
"VN", samtools_version(),
arg_list ? "CL": NULL,
arg_list ? arg_list : NULL,
NULL) != 0)
goto fail;
- free(h->text);
- h->text = strdup(sam_hdr_str(sh));
- h->l_text = sam_hdr_length(sh);
- if (!h->text)
- goto fail;
- sam_hdr_free(sh);
- sh = NULL;
- }
-
if (bam_hdr_write(fp, h) < 0) {
print_error_errno("reheader", "Couldn't write header");
goto fail;
fail:
bgzf_close(fp);
free(buf);
- sam_hdr_free(sh);
return -1;
}
*
* FIXME: error checking
*/
-int cram_reheader(cram_fd *in, bam_hdr_t *h, const char *arg_list, int add_PG)
+int cram_reheader(cram_fd *in, sam_hdr_t *h, const char *arg_list, int no_pg)
{
htsFile *h_out = hts_open("-", "wc");
cram_fd *out = h_out->fp.cram;
cram_container *c = NULL;
int ret = -1;
+ if (!h)
+ return ret;
// Attempt to fill out a cram->refs[] array from @SQ headers
- cram_fd_set_header(out, sam_hdr_parse_(h->text, h->l_text));
- if (add_PG) {
- if (sam_hdr_add_PG(cram_fd_get_header(out), "samtools",
+ sam_hdr_t *cram_h = sam_hdr_dup(h);
+ if (!cram_h)
+ return -1;
+ cram_fd_set_header(out, cram_h);
+ if (!no_pg && sam_hdr_add_pg(cram_fd_get_header(out), "samtools",
"VN", samtools_version(),
arg_list ? "CL": NULL,
arg_list ? arg_list : NULL,
- NULL) != 0)
+ NULL))
goto err;
- // Covert back to bam_hdr_t struct
- free(h->text);
- h->text = strdup(sam_hdr_str(cram_fd_get_header(out)));
- h->l_text = sam_hdr_length(cram_fd_get_header(out));
- if (!h->text)
- goto err;
- }
-
- if (sam_hdr_write(h_out, h) != 0)
+ if (sam_hdr_write(h_out, cram_h) != 0)
goto err;
cram_set_option(out, CRAM_OPT_REFERENCE, NULL);
* -1 on general failure;
* -2 on failure due to insufficient size
*/
-int cram_reheader_inplace2(cram_fd *fd, const bam_hdr_t *h, const char *arg_list,
- int add_PG)
+int cram_reheader_inplace2(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
+ int no_pg)
{
cram_container *c = NULL;
cram_block *b = NULL;
- SAM_hdr *hdr = NULL;
+ sam_hdr_t *cram_h = NULL;
off_t start;
int ret = -1;
+ if (!h)
+ goto err;
if (cram_major_vers(fd) < 2 ||
cram_major_vers(fd) > 3) {
goto err;
}
- if (!(hdr = sam_hdr_parse_(h->text, h->l_text)))
+ cram_h = sam_hdr_dup(h);
+ if (!cram_h)
goto err;
- if (add_PG && sam_hdr_add_PG(hdr, "samtools", "VN", samtools_version(),
+ if (!no_pg && sam_hdr_add_pg(cram_h, "samtools", "VN", samtools_version(),
arg_list ? "CL": NULL,
arg_list ? arg_list : NULL,
NULL))
goto err;
- int header_len = sam_hdr_length(hdr);
+ int header_len = sam_hdr_length(cram_h);
/* Fix M5 strings? Maybe out of scope for this tool */
// Load the existing header
cram_block_set_offset(b, 0); // rewind block
int32_put_blk(b, header_len);
- cram_block_append(b, sam_hdr_str(hdr), header_len);
+ cram_block_append(b, (void *)sam_hdr_str(cram_h), header_len);
// Zero the remaining block
memset((char *)cram_block_get_data(b)+cram_block_get_offset(b), 0,
cram_block_get_uncomp_size(b) - cram_block_get_offset(b));
err:
if (c) cram_free_container(c);
if (b) cram_free_block(b);
- if (hdr) sam_hdr_free(hdr);
+ if (cram_h) sam_hdr_destroy(cram_h);
return ret;
}
* -1 on general failure;
* -2 on failure due to insufficient size
*/
-int cram_reheader_inplace3(cram_fd *fd, const bam_hdr_t *h, const char *arg_list,
- int add_PG)
+int cram_reheader_inplace3(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
+ int no_pg)
{
cram_container *c = NULL;
cram_block *b = NULL;
- SAM_hdr *hdr = NULL;
+ sam_hdr_t *cram_h = NULL;
off_t start, sz, end;
int container_sz, max_container_sz;
char *buf = NULL;
int ret = -1;
+ if (!h)
+ goto err;
if (cram_major_vers(fd) < 2 ||
cram_major_vers(fd) > 3) {
goto err;
}
- if (!(hdr = sam_hdr_parse_(h->text, h->l_text)))
+ cram_h = sam_hdr_dup(h);
+ if (!cram_h)
goto err;
- if (add_PG && sam_hdr_add_PG(hdr, "samtools", "VN", samtools_version(),
+ if (!no_pg && sam_hdr_add_pg(cram_h, "samtools", "VN", samtools_version(),
arg_list ? "CL": NULL,
arg_list ? arg_list : NULL,
NULL))
goto err;
- int header_len = sam_hdr_length(hdr);
+ int header_len = sam_hdr_length(cram_h);
/* Fix M5 strings? Maybe out of scope for this tool */
// Find current size of SAM header block
// Version 3.0 supports compressed header
b = cram_new_block(FILE_HEADER, 0);
int32_put_blk(b, header_len);
- cram_block_append(b, sam_hdr_str(hdr), header_len);
+ cram_block_append(b, (void *)sam_hdr_str(cram_h), header_len);
cram_block_update_size(b);
cram_compress_block(fd, b, NULL, -1, -1);
if (c) cram_free_container(c);
if (buf) free(buf);
if (b) cram_free_block(b);
- if (hdr) sam_hdr_free(hdr);
+ if (cram_h) sam_hdr_destroy(cram_h);
return ret;
}
-int cram_reheader_inplace(cram_fd *fd, const bam_hdr_t *h, const char *arg_list,
- int add_PG)
+int cram_reheader_inplace(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
+ int no_pg)
{
switch (cram_major_vers(fd)) {
- case 2: return cram_reheader_inplace2(fd, h, arg_list, add_PG);
- case 3: return cram_reheader_inplace3(fd, h, arg_list, add_PG);
+ case 2: return cram_reheader_inplace2(fd, h, arg_list, no_pg);
+ case 3: return cram_reheader_inplace3(fd, h, arg_list, no_pg);
default:
fprintf(samtools_stderr, "[%s] unsupported CRAM version %d\n", __func__,
cram_major_vers(fd));
static void usage(FILE *fp, int ret) {
fprintf(fp,
"Usage: samtools reheader [-P] in.header.sam in.bam > out.bam\n"
- " or samtools reheader [-P] -i in.header.sam file.bam\n"
+ " or samtools reheader [-P] -i in.header.sam file.cram\n"
+ " or samtools reheader -c CMD in.bam\n"
+ " or samtools reheader -c CMD in.cram\n"
"\n"
"Options:\n"
- " -P, --no-PG Do not generate an @PG header line.\n"
- " -i, --in-place Modify the bam/cram file directly.\n"
- " (Defaults to outputting to samtools_stdout.)\n");
+ " -P, --no-PG Do not generate a @PG header line.\n"
+ " -i, --in-place Modify the CRAM file directly, if possible.\n"
+ " (Defaults to outputting to samtools_stdout.)\n"
+ " -c, --command CMD Pass the header in SAM format to external program CMD.\n");
exit(ret);
}
+static sam_hdr_t* external_reheader(samFile* in, const char* external) {
+ char *command = NULL;
+ sam_hdr_t* h = NULL;
+ sam_hdr_t* ih = sam_hdr_read(in);
+ if (ih == NULL) {
+ fprintf(samtools_stderr, "[%s] failed to read the header for '%s'.\n", __func__, in->fn);
+ return NULL;
+ }
+ char tmp_fn[] = "reheaderXXXXXX";
+ int tmp_fd = mkstemp(tmp_fn);
+ if (tmp_fd < 0) {
+ print_error_errno("reheader", "fail to open temp file '%s'", tmp_fn);
+ return NULL;
+ }
+ hFILE* tmp_hf = hdopen(tmp_fd, "w");
+ if (!tmp_hf) {
+ fprintf(samtools_stderr, "[%s] failed to convert to hFILE.\n", __func__);
+ goto cleanup;
+ }
+ samFile* tmp_sf = hts_hopen(tmp_hf, tmp_fn, "w");
+ if (!tmp_sf) {
+ fprintf(samtools_stderr, "[%s] failed to convert to samFile.\n", __func__);
+ goto cleanup;
+ }
+ if (-1 == sam_hdr_write(tmp_sf, ih)) {
+ fprintf(samtools_stderr, "[%s] failed to write the header to the temp file.\n", __func__);
+ goto cleanup;
+ }
+ sam_close(tmp_sf);
+ sam_hdr_destroy(ih);
+ int comm_len = strlen(external) + strlen(tmp_fn) + 8;
+ command = calloc(comm_len, 1);
+ if (!command || snprintf(command, comm_len, "( %s ) < %s", external, tmp_fn) != comm_len - 1) {
+ fprintf(samtools_stderr, "[%s] failed to create command string.\n", __func__);
+ goto cleanup;
+ }
+ FILE* nh = popen(command, "r");
+ if (!nh) {
+ print_error_errno("reheader", "[%s] failed to run external command '%s'.\n", __func__, command);
+ goto cleanup;
+ }
+
+ int nh_fd = dup(fileno(nh));
+ if (nh_fd < 0) {
+ fprintf(samtools_stderr, "[%s] failed to get the file descriptor.\n", __func__);
+ goto cleanup;
+ }
+ hFILE* nh_hf = hdopen(nh_fd, "r");
+ if (!nh_hf) {
+ fprintf(samtools_stderr, "[%s] failed to convert to hFILE.\n", __func__);
+ goto cleanup;
+ }
+ samFile* nh_sf = hts_hopen(nh_hf, tmp_fn, "r");
+ if (!nh_sf) {
+ fprintf(samtools_stderr, "[%s] failed to convert to samFile.\n", __func__);
+ goto cleanup;
+ }
+
+ h = sam_hdr_read(nh_sf);
+ sam_close(nh_sf);
+ if (h == NULL) {
+ fprintf(samtools_stderr, "[%s] failed to read the header from the temp file.\n", __func__);
+ }
+ int res = pclose(nh);
+ if (res != 0) {
+ if (res < 0) {
+ print_error_errno("reheader",
+ "Error on closing pipe from command '%s'.\n",
+ command);
+ } else {
+ print_error("reheader",
+ "Non-zero exit code returned by command '%s'\n",
+ command);
+ }
+ if (h) sam_hdr_destroy(h);
+ h = NULL;
+ }
+cleanup:
+ free(command);
+ if (unlink(tmp_fn) != 0) {
+ print_error_errno("reheader", "failed to remove the temp file '%s'", tmp_fn);
+ }
+
+ return h;
+}
+
int main_reheader(int argc, char *argv[])
{
- int inplace = 0, r, add_PG = 1, c;
- bam_hdr_t *h;
+ int inplace = 0, r, no_pg = 0, c, skip_header = 0;
+ sam_hdr_t *h;
samFile *in;
- char *arg_list = stringify_argv(argc+1, argv-1);
+ char *arg_list = NULL, *external = NULL;
static const struct option lopts[] = {
{"help", no_argument, NULL, 'h'},
{"in-place", no_argument, NULL, 'i'},
{"no-PG", no_argument, NULL, 'P'},
+ {"command", required_argument, NULL, 'c'},
{NULL, 0, NULL, 0}
};
- while ((c = getopt_long(argc, argv, "hiP", lopts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "hiPc:", lopts, NULL)) >= 0) {
switch (c) {
- case 'P': add_PG = 0; break;
+ case 'P': no_pg = 1; break;
case 'i': inplace = 1; break;
+ case 'c': external = optarg; break;
case 'h': usage(samtools_stdout, 0); break;
default:
fprintf(samtools_stderr, "Invalid option '%c'\n", c);
}
}
- if (argc - optind != 2)
+ if ((argc - optind != 2 || external) && (argc - optind != 1 || !external))
usage(samtools_stderr, 1);
- { // read the header
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("reheader", "failed to create arg_list");
+ return 1;
+ }
+
+ if (external) {
+ skip_header = 1;
+ in = sam_open(argv[optind], inplace?"r+":"r");
+ if (in == 0) {
+ print_error_errno("reheader", "fail to open file '%s'", argv[optind]);
+ return 1;
+ }
+
+ h = external_reheader(in, external);
+ if (h == NULL) {
+ fprintf(samtools_stderr, "[%s] failed to read the header from '%s'.\n", __func__, external);
+ sam_close(in);
+ return 1;
+ }
+ } else { // read the header from a separate file
samFile *fph = sam_open(argv[optind], "r");
if (fph == 0) {
print_error_errno("reheader", "fail to read the header from '%s'", argv[optind]);
__func__, argv[1]);
return 1;
}
+ in = sam_open(argv[optind+1], inplace?"r+":"r");
+ if (in == 0) {
+ print_error_errno("reheader", "fail to open file '%s'", argv[optind+1]);
+ return 1;
+ }
}
- in = sam_open(argv[optind+1], inplace?"r+":"r");
- if (in == 0) {
- print_error_errno("reheader", "fail to open file '%s'", argv[optind+1]);
- return 1;
- }
+
if (hts_get_format(in)->format == bam) {
- r = bam_reheader(in->fp.bgzf, h, fileno(samtools_stdout), arg_list, add_PG);
- } else {
+ if (inplace) {
+ print_error("reheader", "cannot reheader BAM '%s' in-place", argv[optind+1]);
+ r = -1;
+ } else {
+ r = bam_reheader(in->fp.bgzf, h, fileno(samtools_stdout), arg_list, no_pg, skip_header);
+ }
+ } else if (hts_get_format(in)->format == cram) {
if (inplace)
- r = cram_reheader_inplace(in->fp.cram, h, arg_list, add_PG);
+ r = cram_reheader_inplace(in->fp.cram, h, arg_list, no_pg);
else
- r = cram_reheader(in->fp.cram, h, arg_list, add_PG);
+ r = cram_reheader(in->fp.cram, h, arg_list, no_pg);
+ } else {
+ print_error("reheader", "input file '%s' must be BAM or CRAM", argv[optind+1]);
+ r = -1;
}
if (sam_close(in) != 0)
r = -1;
- bam_hdr_destroy(h);
+ sam_hdr_destroy(h);
if (arg_list)
free(arg_list);
/* bam_rmdup.c -- duplicate read detection.
- Copyright (C) 2009, 2015 Genome Research Ltd.
+ Copyright (C) 2009, 2015, 2016, 2019 Genome Research Ltd.
Portions copyright (C) 2009 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
stack->a[stack->n++] = b;
}
-static inline int dump_best(tmp_stack_t *stack, samFile *out, bam_hdr_t *hdr)
+static inline int dump_best(tmp_stack_t *stack, samFile *out, sam_hdr_t *hdr)
{
int i;
for (i = 0; i != stack->n; ++i) {
return q;
}
-int bam_rmdup_core(samFile *in, bam_hdr_t *hdr, samFile *out)
+int bam_rmdup_core(samFile *in, sam_hdr_t *hdr, samFile *out)
{
bam1_t *b = NULL;
int last_tid = -1, last_pos = -1, r;
break;
}
last_tid = c->tid;
- fprintf(stderr, "[bam_rmdup_core] processing reference %s...\n", hdr->target_name[c->tid]);
+ fprintf(stderr, "[bam_rmdup_core] processing reference %s...\n", sam_hdr_tid2name(hdr, c->tid));
}
}
if (!(c->flag&BAM_FPAIRED) || (c->flag&(BAM_FUNMAP|BAM_FMUNMAP)) || (c->mtid >= 0 && c->tid != c->mtid)) {
q = lib? get_aux(aux, lib) : get_aux(aux, "\t");
++q->n_checked;
k = kh_put(pos, q->best_hash, key, &ret);
+ if (ret < 0) goto fail;
if (ret == 0) { // found in best_hash
bam1_t *p = kh_val(q->best_hash, k);
++q->n_removed;
if (sum_qual(p) < sum_qual(b)) { // the current alignment is better; this can be accelerated in principle
kh_put(name, del_set, strdup(bam_get_qname(p)), &ret); // p will be removed
- bam_copy1(p, b); // replaced as b
+ if (ret < 0) goto fail;
+ if (bam_copy1(p, b) == NULL) goto fail; // replaced as b
} else kh_put(name, del_set, strdup(bam_get_qname(b)), &ret); // b will be removed
+ if (ret < 0) goto fail;
if (ret == 0)
fprintf(stderr, "[bam_rmdup_core] inconsistent BAM file for pair '%s'. Continue anyway.\n", bam_get_qname(b));
} else { // not found in best_hash
return 1;
}
-int bam_rmdupse_core(samFile *in, bam_hdr_t *hdr, samFile *out, int force_se);
+int bam_rmdupse_core(samFile *in, sam_hdr_t *hdr, samFile *out, int force_se);
static int rmdup_usage(void) {
fprintf(stderr, "\n");
fprintf(stderr, "Option: -s rmdup for SE reads\n");
fprintf(stderr, " -S treat PE reads as SE in rmdup (force -s)\n");
- sam_global_opt_help(stderr, "-....-");
+ sam_global_opt_help(stderr, "-....--.");
return 1;
}
{
int c, ret, is_se = 0, force_se = 0;
samFile *in, *out;
- bam_hdr_t *header;
+ sam_hdr_t *header;
char wmode[3] = {'w', 'b', 0};
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
return 1;
}
header = sam_hdr_read(in);
- if (header == NULL || header->n_targets == 0) {
+ if (header == NULL || sam_hdr_nref(header) == 0) {
fprintf(stderr, "[bam_rmdup] input SAM does not have header. Abort!\n");
return 1;
}
if (is_se) ret = bam_rmdupse_core(in, header, out, force_se);
else ret = bam_rmdup_core(in, header, out);
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
sam_close(in);
if (sam_close(out) < 0) {
fprintf(stderr, "[bam_rmdup] error closing output file\n");
/* bam_rmdup.c -- duplicate read detection.
- Copyright (C) 2009, 2015 Genome Research Ltd.
+ Copyright (C) 2009, 2015, 2016, 2019 Genome Research Ltd.
Portions copyright (C) 2009 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
stack->a[stack->n++] = b;
}
-static inline int dump_best(tmp_stack_t *stack, samFile *out, bam_hdr_t *hdr)
+static inline int dump_best(tmp_stack_t *stack, samFile *out, sam_hdr_t *hdr)
{
int i;
for (i = 0; i != stack->n; ++i) {
return q;
}
-int bam_rmdup_core(samFile *in, bam_hdr_t *hdr, samFile *out)
+int bam_rmdup_core(samFile *in, sam_hdr_t *hdr, samFile *out)
{
bam1_t *b = NULL;
int last_tid = -1, last_pos = -1, r;
break;
}
last_tid = c->tid;
- fprintf(samtools_stderr, "[bam_rmdup_core] processing reference %s...\n", hdr->target_name[c->tid]);
+ fprintf(samtools_stderr, "[bam_rmdup_core] processing reference %s...\n", sam_hdr_tid2name(hdr, c->tid));
}
}
if (!(c->flag&BAM_FPAIRED) || (c->flag&(BAM_FUNMAP|BAM_FMUNMAP)) || (c->mtid >= 0 && c->tid != c->mtid)) {
q = lib? get_aux(aux, lib) : get_aux(aux, "\t");
++q->n_checked;
k = kh_put(pos, q->best_hash, key, &ret);
+ if (ret < 0) goto fail;
if (ret == 0) { // found in best_hash
bam1_t *p = kh_val(q->best_hash, k);
++q->n_removed;
if (sum_qual(p) < sum_qual(b)) { // the current alignment is better; this can be accelerated in principle
kh_put(name, del_set, strdup(bam_get_qname(p)), &ret); // p will be removed
- bam_copy1(p, b); // replaced as b
+ if (ret < 0) goto fail;
+ if (bam_copy1(p, b) == NULL) goto fail; // replaced as b
} else kh_put(name, del_set, strdup(bam_get_qname(b)), &ret); // b will be removed
+ if (ret < 0) goto fail;
if (ret == 0)
fprintf(samtools_stderr, "[bam_rmdup_core] inconsistent BAM file for pair '%s'. Continue anyway.\n", bam_get_qname(b));
} else { // not found in best_hash
return 1;
}
-int bam_rmdupse_core(samFile *in, bam_hdr_t *hdr, samFile *out, int force_se);
+int bam_rmdupse_core(samFile *in, sam_hdr_t *hdr, samFile *out, int force_se);
static int rmdup_usage(void) {
fprintf(samtools_stderr, "\n");
fprintf(samtools_stderr, "Option: -s rmdup for SE reads\n");
fprintf(samtools_stderr, " -S treat PE reads as SE in rmdup (force -s)\n");
- sam_global_opt_help(samtools_stderr, "-....-");
+ sam_global_opt_help(samtools_stderr, "-....--.");
return 1;
}
{
int c, ret, is_se = 0, force_se = 0;
samFile *in, *out;
- bam_hdr_t *header;
+ sam_hdr_t *header;
char wmode[3] = {'w', 'b', 0};
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
return 1;
}
header = sam_hdr_read(in);
- if (header == NULL || header->n_targets == 0) {
+ if (header == NULL || sam_hdr_nref(header) == 0) {
fprintf(samtools_stderr, "[bam_rmdup] input SAM does not have header. Abort!\n");
return 1;
}
if (is_se) ret = bam_rmdupse_core(in, header, out, force_se);
else ret = bam_rmdup_core(in, header, out);
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
sam_close(in);
if (sam_close(out) < 0) {
fprintf(samtools_stderr, "[bam_rmdup] error closing output file\n");
/* bam_rmdupse.c -- duplicate read detection for unpaired reads.
- Copyright (C) 2009, 2015 Genome Research Ltd.
+ Copyright (C) 2009, 2015, 2016, 2019 Genome Research Ltd.
Portions copyright (C) 2009 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
p->discarded = 0;
p->endpos = endpos; p->score = score;
if (p->b == 0) p->b = bam_init1();
- bam_copy1(p->b, b);
+ if (!p->b) { perror(NULL); exit(EXIT_FAILURE); }
+ if (bam_copy1(p->b, b) == NULL) { perror(NULL); exit(EXIT_FAILURE); }
return p;
}
kh_del(best, h, k);
}
-static int dump_alignment(samFile *out, bam_hdr_t *hdr,
+static int dump_alignment(samFile *out, sam_hdr_t *hdr,
queue_t *queue, int32_t pos, khash_t(lib) *h)
{
if (queue->size > QUEUE_CLEAR_SIZE || pos == MAX_POS) {
return 0;
}
-int bam_rmdupse_core(samFile *in, bam_hdr_t *hdr, samFile *out, int force_se)
+int bam_rmdupse_core(samFile *in, sam_hdr_t *hdr, samFile *out, int force_se)
{
bam1_t *b = NULL;
queue_t *queue = NULL;
kh_val(h, k) = push_queue(queue, b, endpos, score);
} else { // replace
p->score = score; p->endpos = endpos;
- bam_copy1(p->b, b);
+ if (bam_copy1(p->b, b) == NULL) {
+ perror(NULL); exit(EXIT_FAILURE);
+ }
}
} // otherwise, discard the alignment
} else kh_val(h, k) = push_queue(queue, b, endpos, score);
/* bam_rmdupse.c -- duplicate read detection for unpaired reads.
- Copyright (C) 2009, 2015 Genome Research Ltd.
+ Copyright (C) 2009, 2015, 2016, 2019 Genome Research Ltd.
Portions copyright (C) 2009 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
p->discarded = 0;
p->endpos = endpos; p->score = score;
if (p->b == 0) p->b = bam_init1();
- bam_copy1(p->b, b);
+ if (!p->b) { perror(NULL); exit(EXIT_FAILURE); }
+ if (bam_copy1(p->b, b) == NULL) { perror(NULL); exit(EXIT_FAILURE); }
return p;
}
kh_del(best, h, k);
}
-static int dump_alignment(samFile *out, bam_hdr_t *hdr,
+static int dump_alignment(samFile *out, sam_hdr_t *hdr,
queue_t *queue, int32_t pos, khash_t(lib) *h)
{
if (queue->size > QUEUE_CLEAR_SIZE || pos == MAX_POS) {
return 0;
}
-int bam_rmdupse_core(samFile *in, bam_hdr_t *hdr, samFile *out, int force_se)
+int bam_rmdupse_core(samFile *in, sam_hdr_t *hdr, samFile *out, int force_se)
{
bam1_t *b = NULL;
queue_t *queue = NULL;
kh_val(h, k) = push_queue(queue, b, endpos, score);
} else { // replace
p->score = score; p->endpos = endpos;
- bam_copy1(p->b, b);
+ if (bam_copy1(p->b, b) == NULL) {
+ perror(NULL); exit(EXIT_FAILURE);
+ }
}
} // otherwise, discard the alignment
} else kh_val(h, k) = push_queue(queue, b, endpos, score);
/* bam_sort.c -- sorting and merging.
- Copyright (C) 2008-2016 Genome Research Ltd.
+ Copyright (C) 2008-2019 Genome Research Ltd.
Portions copyright (C) 2009-2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include "htslib/klist.h"
#include "htslib/kstring.h"
#include "htslib/sam.h"
+#include "htslib/hts_endian.h"
#include "sam_opts.h"
#include "samtools.h"
bam1_t *bam_record;
union {
const uint8_t *tag;
- uint64_t pos;
+ uint8_t pos_tid[12];
} u;
} bam1_tag;
return *pa? 1 : *pb? -1 : 0;
}
-#define HEAP_EMPTY UINT64_MAX
+#define HEAP_EMPTY (UINT64_MAX >> 1)
typedef struct {
int i;
- uint32_t rev;
- uint64_t pos, idx;
+ uint32_t tid;
+ uint64_t pos:63, rev:1, idx;
bam1_tag entry;
} heap1_t;
fb = b.entry.bam_record->core.flag & 0xc0;
if (fa != fb) return fa > fb;
} else {
+ if (a.tid != b.tid) return a.tid > b.tid;
if (a.pos != b.pos) return a.pos > b.pos;
if (a.rev != b.rev) return a.rev > b.rev;
}
KSORT_INIT(heap, heap1_t, heap_lt)
typedef struct merged_header {
- kstring_t out_hd;
- kstring_t out_sq;
+ sam_hdr_t *hdr;
kstring_t out_rg;
kstring_t out_pg;
kstring_t out_co;
bool lost_coord_sort;
} trans_tbl_t;
-/* Something to look like a regmatch_t */
-typedef struct hdr_match {
- ptrdiff_t rm_so;
- ptrdiff_t rm_eo;
-} hdr_match_t;
-
-/*
- * Search for header lines of a particular record type.
- *
- * This replaces a regex search for something like /^@SQ.*\tSN:([^\t]+).*$/
- * but is much quicker. The locations found are returned in *matches,
- * which has a signature the same as that of a regmatch_t.
- *
- * rec is the record type to match (i.e. @HD, @SQ, @PG or @RG)
- * tag is a tag type in the record to match (SN for @SQ, ID for @PG or @RG)
- *
- * The location of the record (if found) is returned in matches[0]
- * If tag is not NULL, the record is searched for the presence of the
- * given tag. If found, the location of the value is returned in matches[1].
- * If the tag isn't found then the record is ignored and the search resumes
- * on the next header line.
- *
- * For simplicity, some assumptions are made about rec and tag:
- * rec should include the leading '@' sign and be three characters long.
- * tag should be exactly two characters long.
- * These are always string constants when this is called below, so we don't
- * bother to check here.
- *
- * Returns 0 if a match was found, -1 if not.
- */
-
-
-static int hdr_line_match(const char *text, const char *rec,
- const char *tag, hdr_match_t *matches) {
- const char *line_start, *line_end = text;
- const char *tag_start, *tag_end;
-
- for (;;) {
- // Find record, ensure either at start of text or follows '\n'
- line_start = strstr(line_end, rec);
- while (line_start && line_start > text && *(line_start - 1) != '\n') {
- line_start = strstr(line_start + 3, rec);
- }
- if (!line_start) return -1;
-
- // Find end of header line
- line_end = strchr(line_start, '\n');
- if (!line_end) line_end = line_start + strlen(line_start);
-
- matches[0].rm_so = line_start - text;
- matches[0].rm_eo = line_end - text;
- if (!tag) return 0; // Match found if not looking for tag.
-
- for (tag_start = line_start + 3; tag_start < line_end; tag_start++) {
- // Find possible tag start. Hacky but quick.
- while (*tag_start > '\n') tag_start++;
-
- // Check it
- if (tag_start[0] == '\t'
- && strncmp(tag_start + 1, tag, 2) == 0
- && tag_start[3] == ':') {
- // Found tag, record location and return.
- tag_end = tag_start + 4;
- while (*tag_end && *tag_end != '\t' && *tag_end != '\n')
- ++tag_end;
- matches[1].rm_so = tag_start - text + 4;
- matches[1].rm_eo = tag_end - text;
- return 0;
- }
- }
- // Couldn't find tag, try again from end of current record.
- }
-}
-
static void trans_tbl_destroy(trans_tbl_t *tbl) {
khiter_t iter;
merged_hdr = calloc(1, sizeof(*merged_hdr));
if (merged_hdr == NULL) return NULL;
+ merged_hdr->hdr = sam_hdr_init();
+ if (!merged_hdr->hdr) goto fail;
+
merged_hdr->targets_sz = 16;
merged_hdr->target_name = malloc(merged_hdr->targets_sz
* sizeof(*merged_hdr->target_name));
kh_destroy(c2i, merged_hdr->sq_tids);
free(merged_hdr->target_name);
free(merged_hdr->target_len);
+ sam_hdr_destroy(merged_hdr->hdr);
free(merged_hdr);
return NULL;
}
return kputsn(src + from, to - from, dest) != to - from;
}
-// Append a header line match to kstring
-static inline int match_to_ks(const char *src, const hdr_match_t *match,
- kstring_t *dest) {
- return range_to_ks(src, match->rm_so, match->rm_eo, dest);
-}
-
// Append a kstring to a kstring
static inline int ks_to_ks(kstring_t *src, kstring_t *dest) {
return kputsn(ks_str(src), ks_len(src), dest) != ks_len(src);
*/
static int trans_tbl_add_hd(merged_header_t* merged_hdr,
- bam_hdr_t *translate) {
- hdr_match_t match = {0, 0};
+ sam_hdr_t *translate) {
+ kstring_t hd_line = { 0, 0, NULL };
+ int res;
// TODO: handle case when @HD needs merging.
if (merged_hdr->have_hd) return 0;
- if (hdr_line_match(translate->text, "@HD", NULL, &match) != 0) {
- return 0;
+ res = sam_hdr_find_hd(translate, &hd_line);
+ if (res < -1) {
+ print_error("merge", "failed to get @HD line from header");
+ return -1;
}
- if (match_to_ks(translate->text, &match, &merged_hdr->out_hd)) goto memfail;
- if (kputc('\n', &merged_hdr->out_hd) == EOF) goto memfail;
- merged_hdr->have_hd = true;
-
- return 0;
-
- memfail:
- perror(__func__);
- return -1;
-}
-
-static inline int grow_target_list(merged_header_t* merged_hdr) {
- size_t new_size;
- char **new_names;
- uint32_t *new_len;
-
- new_size = merged_hdr->targets_sz * 2;
- new_names = realloc(merged_hdr->target_name, sizeof(*new_names) * new_size);
- if (!new_names) goto fail;
- merged_hdr->target_name = new_names;
+ if (res < 0) // Not found
+ return 0;
- new_len = realloc(merged_hdr->target_len, sizeof(*new_len) * new_size);
- if (!new_len) goto fail;
- merged_hdr->target_len = new_len;
+ if (sam_hdr_add_lines(merged_hdr->hdr, hd_line.s, hd_line.l) < 0) {
+ print_error("merge", "failed to add @HD line to new header");
+ free(hd_line.s);
+ return -1;
+ }
- merged_hdr->targets_sz = new_size;
+ free(hd_line.s);
+ merged_hdr->have_hd = true;
return 0;
-
- fail:
- perror(__func__);
- return -1;
}
/*
* Returns 0 on success, -1 on failure.
*/
-static int trans_tbl_add_sq(merged_header_t* merged_hdr, bam_hdr_t *translate,
+static int trans_tbl_add_sq(merged_header_t* merged_hdr, sam_hdr_t *translate,
trans_tbl_t* tbl) {
-
- kstring_t *out_text = &merged_hdr->out_sq;
- khash_t(c2i)* sq_tids = merged_hdr->sq_tids;
- hdr_match_t *new_sq_matches = NULL;
- char *text;
- hdr_match_t matches[2];
int32_t i;
- int32_t old_n_targets = merged_hdr->n_targets;
- khiter_t iter;
- int min_tid = -1;
+ int min_tid = -1, res;
+ kstring_t sq_line = { 0, 0, NULL }, sq_sn = { 0, 0, NULL };
// Fill in the tid part of the translation table, adding new targets
// to the merged header as we go.
- for (i = 0; i < translate->n_targets; ++i) {
+ for (i = 0; i < sam_hdr_nref(translate); ++i) {
+ int trans_tid;
+ sq_sn.l = 0;
+ res = sam_hdr_find_tag_pos(translate, "SQ", i, "SN", &sq_sn);
+ if (res < 0) {
+ print_error("merge", "failed to get @SQ SN #%d from header", i + 1);
+ goto fail;
+ }
- // Check if it's a new target.
- iter = kh_get(c2i, sq_tids, translate->target_name[i]);
+ trans_tid = sam_hdr_name2tid(merged_hdr->hdr, sq_sn.s);
+ if (trans_tid < -1) {
+ print_error("merge", "failed to lookup ref");
+ goto fail;
+ }
- if (iter == kh_end(sq_tids)) {
- int ret;
+ if (trans_tid < 0) {
// Append missing entries to out_hdr
-
- if (merged_hdr->n_targets == merged_hdr->targets_sz) {
- if (grow_target_list(merged_hdr)) goto fail;
+ sq_line.l = 0;
+ res = sam_hdr_find_line_id(translate, "SQ", "SN", sq_sn.s, &sq_line);
+ if (res < 0) {
+ print_error("merge", "failed to get @SQ SN:%s from header", sq_sn.s);
+ goto fail;
}
- merged_hdr->target_name[merged_hdr->n_targets] = strdup(translate->target_name[i]);
- if (merged_hdr->target_name[merged_hdr->n_targets] == NULL) goto memfail;
- merged_hdr->target_len[merged_hdr->n_targets] = translate->target_len[i];
+ trans_tid = sam_hdr_nref(merged_hdr->hdr);
- // Record the new identifier for reference below,
- // and when building the ttable for other inputs.
- iter = kh_put(c2i, sq_tids,
- merged_hdr->target_name[merged_hdr->n_targets], &ret);
- if (ret < 0) {
- free(merged_hdr->target_name[merged_hdr->n_targets]);
- goto memfail;
+ res = sam_hdr_add_lines(merged_hdr->hdr, sq_line.s, sq_line.l);
+ if (res < 0) {
+ print_error("merge", "failed to add @SQ SN:%s to new header", sq_sn.s);
+ goto fail;
}
- assert(ret > 0); // Should not be in hash already.
-
- kh_value(sq_tids, iter) = merged_hdr->n_targets;
- tbl->tid_trans[i] = merged_hdr->n_targets++;
- } else {
- tbl->tid_trans[i] = kh_value(sq_tids, iter);
}
+ tbl->tid_trans[i] = trans_tid;
if (tbl->tid_trans[i] > min_tid) {
min_tid = tbl->tid_trans[i];
}
}
- if (merged_hdr->n_targets == old_n_targets)
- return 0; // Everything done if no new targets.
-
- // Otherwise, find @SQ lines in translate->text for all newly added targets.
-
- new_sq_matches = malloc((merged_hdr->n_targets - old_n_targets)
- * sizeof(*new_sq_matches));
- if (new_sq_matches == NULL) goto memfail;
-
- for (i = 0; i < merged_hdr->n_targets - old_n_targets; i++) {
- new_sq_matches[i].rm_so = new_sq_matches[i].rm_eo = -1;
- }
-
- text = translate->text;
- while (hdr_line_match(text, "@SQ", "SN", matches) == 0) {
- // matches[0] is whole line, matches[1] is SN value.
-
- // This is a bit disgusting, but avoids a copy...
- char c = text[matches[1].rm_eo];
- int idx;
-
- text[matches[1].rm_eo] = '\0';
-
- // Look up the SN value in the sq_tids hash.
- iter = kh_get(c2i, sq_tids, text + matches[1].rm_so);
- text[matches[1].rm_eo] = c; // restore text
-
- if (iter == kh_end(sq_tids)) {
- // Warn about this, but it's not really fatal.
- fprintf(stderr, "[W::%s] @SQ SN (%.*s) found in text header but not binary header.\n",
- __func__,
- (int) (matches[1].rm_eo - matches[1].rm_so),
- text + matches[1].rm_so);
- text += matches[0].rm_eo;
- continue; // Skip to next
- }
-
- idx = kh_value(sq_tids, iter);
- if (idx >= old_n_targets) {
- // is a new SQ, so record position so we can add it to out_text.
- assert(idx < merged_hdr->n_targets);
- ptrdiff_t off = text - translate->text;
- new_sq_matches[idx - old_n_targets].rm_so = matches[0].rm_so + off;
- new_sq_matches[idx - old_n_targets].rm_eo = matches[0].rm_eo + off;
- }
-
- // Carry on searching from end of current match
- text += matches[0].rm_eo;
- }
-
- // Copy the @SQ headers found and recreate any missing from binary header.
- for (i = 0; i < merged_hdr->n_targets - old_n_targets; i++) {
- if (new_sq_matches[i].rm_so >= 0) {
- if (match_to_ks(translate->text, &new_sq_matches[i], out_text))
- goto memfail;
- if (kputc('\n', out_text) == EOF) goto memfail;
- } else {
- if (kputs("@SQ\tSN:", out_text) == EOF ||
- kputs(merged_hdr->target_name[i + old_n_targets], out_text) == EOF ||
- kputs("\tLN:", out_text) == EOF ||
- kputuw(merged_hdr->target_len[i + old_n_targets], out_text) == EOF ||
- kputc('\n', out_text) == EOF) goto memfail;
- }
- }
+ free(sq_line.s);
+ free(sq_sn.s);
- free(new_sq_matches);
return 0;
- memfail:
- perror(__func__);
fail:
- free(new_sq_matches);
+ free(sq_line.s);
+ free(sq_sn.s);
return -1;
}
*
*/
-static klist_t(hdrln) * trans_rg_pg(bool is_rg, bam_hdr_t *translate,
+static klist_t(hdrln) * trans_rg_pg(bool is_rg, sam_hdr_t *translate,
bool merge, khash_t(cset)* known_ids,
khash_t(c2c)* id_map, char *override) {
- hdr_match_t matches[2];
khiter_t iter;
- const char *text = translate->text;
- const char *rec_type = is_rg ? "@RG" : "@PG";
+ int num_ids, i;
+ const char *rec_type = is_rg ? "RG" : "PG";
klist_t(hdrln) *hdr_lines;
hdr_lines = kl_init(hdrln);
// Search through translate's header
- while (hdr_line_match(text, rec_type, "ID", matches) == 0) {
- // matches[0] is the whole @RG/PG line; matches[1] is the ID field value
+ num_ids = sam_hdr_count_lines(translate, rec_type);
+ if (num_ids < 0)
+ goto fail;
+ for (i = 0; i < num_ids; i++) {
kstring_t orig_id = { 0, 0, NULL }; // ID in original header
kstring_t transformed_id = { 0, 0, NULL }; // ID in output header
char *map_value; // Value to store in id_map
bool id_changed; // Have we changed the ID?
bool not_found_in_output; // ID isn't in the output header (yet)
- // Take a copy of the ID as we'll need it for a hash key.
- if (match_to_ks(text, &matches[1], &orig_id)) goto memfail;
+ if (sam_hdr_find_tag_pos(translate, rec_type, i, "ID", &orig_id) < 0)
+ goto fail;
// is our matched ID in our output ID set already?
iter = kh_get(cset, known_ids, ks_str(&orig_id));
// Does this line need to go into our output header?
if (not_found_in_output) {
-
// Take matched line and replace ID with transformed_id
kstring_t new_hdr_line = { 0, 0, NULL };
+ if (sam_hdr_find_line_id(translate, rec_type,
+ "ID", ks_str(&orig_id), &new_hdr_line) < 0){
+ goto fail;
+ }
- if (!id_changed) { // Can just copy
- if (match_to_ks(text, &matches[0], &new_hdr_line)) goto memfail;
- } else { // Substitute new name for original
- if (range_to_ks(text, matches[0].rm_so, matches[1].rm_so,
- &new_hdr_line)) goto memfail;
- if (ks_to_ks(&transformed_id, &new_hdr_line)) goto memfail;
- if (range_to_ks(text, matches[1].rm_eo, matches[0].rm_eo,
- &new_hdr_line)) goto memfail;
+ if (id_changed) {
+ char *idp = strstr(ks_str(&new_hdr_line), "\tID:"), *id_end;
+ ptrdiff_t id_offset, id_len;
+ if (!idp) {
+ print_error("merge", "failed to find ID in \"%s\"\n",
+ ks_str(&new_hdr_line));
+ goto fail;
+ }
+ idp += 4;
+ for (id_end = idp; *id_end >= '\n'; id_end++) {}
+
+ id_offset = idp - new_hdr_line.s;
+ id_len = id_end - idp;
+
+ if (id_len < transformed_id.l) {
+ if (ks_resize(&new_hdr_line, new_hdr_line.l + transformed_id.l - id_len))
+ goto fail;
+ }
+ if (id_len != transformed_id.l) {
+ memmove(new_hdr_line.s + id_offset + transformed_id.l,
+ new_hdr_line.s + id_offset + id_len,
+ new_hdr_line.l - id_offset - id_len + 1);
+ }
+ memcpy(new_hdr_line.s + id_offset, transformed_id.s,
+ transformed_id.l);
}
// append line to output linked list
int in_there = 0;
iter = kh_put(c2c, id_map, ks_release(&orig_id), &in_there);
kh_value(id_map, iter) = map_value;
-
- text += matches[0].rm_eo; // next!
}
// If there are no RG lines in the file and we are overriding add one
memfail:
perror(__func__);
+ fail:
if (hdr_lines) kl_destroy(hdrln, hdr_lines);
return NULL;
}
* Returns 0 on success, -1 on failure.
*/
-static int trans_tbl_init(merged_header_t* merged_hdr, bam_hdr_t* translate,
+static int trans_tbl_init(merged_header_t* merged_hdr, sam_hdr_t* translate,
trans_tbl_t* tbl, bool merge_rg, bool merge_pg,
bool copy_co, char* rg_override)
{
+ kstring_t lines = { 0, 0, NULL };
klist_t(hdrln) *rg_list = NULL;
klist_t(hdrln) *pg_list = NULL;
- tbl->n_targets = translate->n_targets;
+ tbl->n_targets = sam_hdr_nref(translate);
tbl->rg_trans = tbl->pg_trans = NULL;
- tbl->tid_trans = (int*)calloc(translate->n_targets, sizeof(int));
+ tbl->tid_trans = (int*)calloc(tbl->n_targets ? tbl->n_targets : 1,
+ sizeof(int));
if (tbl->tid_trans == NULL) goto memfail;
tbl->rg_trans = kh_init(c2c);
if (tbl->rg_trans == NULL) goto memfail;
goto fail;
// Fix-up PP: tags in the new @PG records and add to output
+ lines.l = 0;
if (finish_rg_pg(false, pg_list, tbl->pg_trans, &merged_hdr->out_pg))
goto fail;
if (copy_co) {
// Just append @CO headers without translation
- const char *line, *end_pointer;
- for (line = translate->text; *line; line = end_pointer + 1) {
- end_pointer = strchr(line, '\n');
- if (strncmp(line, "@CO", 3) == 0) {
- if (end_pointer) {
- if (kputsn(line, end_pointer - line + 1, &merged_hdr->out_co) == EOF)
- goto memfail;
- } else { // Last line with no trailing '\n'
- if (kputs(line, &merged_hdr->out_co) == EOF) goto memfail;
- if (kputc('\n', &merged_hdr->out_co) == EOF) goto memfail;
- }
- }
- if (end_pointer == NULL) break;
+ int num_co = sam_hdr_count_lines(translate, "CO"), i;
+ if (num_co < 0)
+ goto fail;
+
+ for (i = 0; i < num_co; i++) {
+ if (sam_hdr_find_line_pos(translate, "CO", i, &lines) < 0)
+ goto fail;
+ if (ks_to_ks(&lines, &merged_hdr->out_co))
+ goto fail;
+ if (kputc('\n', &merged_hdr->out_co) < 0)
+ goto fail;
}
}
+ free(lines.s);
+
return 0;
memfail:
trans_tbl_destroy(tbl);
if (rg_list) kl_destroy(hdrln, rg_list);
if (pg_list) kl_destroy(hdrln, pg_list);
+ free(lines.s);
return -1;
}
-static inline void move_kstr_to_text(char **text, kstring_t *ks) {
- memcpy(*text, ks_str(ks), ks_len(ks));
- *text += ks_len(ks);
- **text = '\0';
- free(ks_release(ks));
-}
-
-/*
- * Populate a bam_hdr_t struct from data in a merged_header_t.
- */
-
-static bam_hdr_t * finish_merged_header(merged_header_t *merged_hdr) {
- size_t txt_sz;
- char *text;
- bam_hdr_t *hdr;
-
- // Check output text size
- txt_sz = (ks_len(&merged_hdr->out_hd)
- + ks_len(&merged_hdr->out_sq)
- + ks_len(&merged_hdr->out_rg)
- + ks_len(&merged_hdr->out_pg)
- + ks_len(&merged_hdr->out_co));
- if (txt_sz >= INT32_MAX) {
- fprintf(stderr, "[%s] Output header text too long\n", __func__);
- return NULL;
- }
-
- // Allocate new header
- hdr = bam_hdr_init();
- if (hdr == NULL) goto memfail;
-
- // Transfer targets arrays to new header
- hdr->n_targets = merged_hdr->n_targets;
- if (hdr->n_targets > 0) {
- // Try to shrink targets arrays to correct size
- hdr->target_name = realloc(merged_hdr->target_name,
- hdr->n_targets * sizeof(char*));
- if (!hdr->target_name) hdr->target_name = merged_hdr->target_name;
-
- hdr->target_len = realloc(merged_hdr->target_len,
- hdr->n_targets * sizeof(uint32_t));
- if (!hdr->target_len) hdr->target_len = merged_hdr->target_len;
-
- // These have either been freed by realloc() or, in the unlikely
- // event that failed, have had their ownership transferred to hdr
- merged_hdr->target_name = NULL;
- merged_hdr->target_len = NULL;
- }
- else {
- hdr->target_name = NULL;
- hdr->target_len = NULL;
- }
-
- // Allocate text
- text = hdr->text = malloc(txt_sz + 1);
- if (!text) goto memfail;
-
- // Put header text in order @HD, @SQ, @RG, @PG, @CO
- move_kstr_to_text(&text, &merged_hdr->out_hd);
- move_kstr_to_text(&text, &merged_hdr->out_sq);
- move_kstr_to_text(&text, &merged_hdr->out_rg);
- move_kstr_to_text(&text, &merged_hdr->out_pg);
- move_kstr_to_text(&text, &merged_hdr->out_co);
- hdr->l_text = txt_sz;
-
- return hdr;
+static int finish_merged_header(merged_header_t *merged_hdr) {
+ if (sam_hdr_add_lines(merged_hdr->hdr, ks_c_str(&merged_hdr->out_rg),
+ ks_len(&merged_hdr->out_rg)) < 0)
+ return -1;
+ if (sam_hdr_add_lines(merged_hdr->hdr, ks_c_str(&merged_hdr->out_pg),
+ ks_len(&merged_hdr->out_pg)) < 0)
+ return -1;
+ if (sam_hdr_add_lines(merged_hdr->hdr, ks_c_str(&merged_hdr->out_co),
+ ks_len(&merged_hdr->out_co)) < 0)
+ return -1;
- memfail:
- perror(__func__);
- bam_hdr_destroy(hdr);
- return NULL;
+ return 0;
}
/*
size_t i;
khiter_t iter;
if (!merged_hdr) return;
- free(ks_release(&merged_hdr->out_hd));
- free(ks_release(&merged_hdr->out_sq));
free(ks_release(&merged_hdr->out_rg));
free(ks_release(&merged_hdr->out_pg));
free(ks_release(&merged_hdr->out_co));
@param cmd command name (used in print_error() etc)
@param in_fmt format options for input files
@param out_fmt output file format and options
+ @param write_index create the index, together with the output file
+ @param arg_list command string for PG line
+ @param no_pg if 1, do not add a new PG line
@discussion Padding information may NOT correctly maintained. This
function is NOT thread safe.
*/
int bam_merge_core2(int by_qname, char* sort_tag, const char *out, const char *mode,
- const char *headers, int n, char * const *fn, int flag,
- const char *reg, int n_threads, const char *cmd,
- const htsFormat *in_fmt, const htsFormat *out_fmt)
+ const char *headers, int n, char * const *fn, char * const *fn_idx,
+ int flag, const char *reg, int n_threads, const char *cmd,
+ const htsFormat *in_fmt, const htsFormat *out_fmt, int write_index,
+ char *arg_list, int no_pg)
{
samFile *fpout, **fp = NULL;
heap1_t *heap = NULL;
- bam_hdr_t *hout = NULL;
- bam_hdr_t *hin = NULL;
+ sam_hdr_t *hout = NULL;
+ sam_hdr_t *hin = NULL;
int i, j, *RG_len = NULL;
uint64_t idx = 0;
char **RG = NULL;
hts_itr_t **iter = NULL;
- bam_hdr_t **hdr = NULL;
+ sam_hdr_t **hdr = NULL;
trans_tbl_t *translation_tbl = NULL;
int *rtrans = NULL;
+ char *out_idx_fn = NULL;
merged_header_t *merged_hdr = init_merged_header();
if (!merged_hdr) return -1;
if (sort_tag) {
g_is_by_tag = 1;
g_sort_tag[0] = sort_tag[0];
- g_sort_tag[1] = sort_tag[1];
+ g_sort_tag[1] = sort_tag[0] ? sort_tag[1] : '\0';
}
fp = (samFile**)calloc(n, sizeof(samFile*));
if (!heap) goto mem_fail;
iter = (hts_itr_t**)calloc(n, sizeof(hts_itr_t*));
if (!iter) goto mem_fail;
- hdr = (bam_hdr_t**)calloc(n, sizeof(bam_hdr_t*));
+ hdr = (sam_hdr_t**)calloc(n, sizeof(sam_hdr_t*));
if (!hdr) goto mem_fail;
translation_tbl = (trans_tbl_t*)calloc(n, sizeof(trans_tbl_t));
if (!translation_tbl) goto mem_fail;
// open and read the header from each file
for (i = 0; i < n; ++i) {
- bam_hdr_t *hin;
+ sam_hdr_t *hin;
fp[i] = sam_open_format(fn[i], "r", in_fmt);
if (fp[i] == NULL) {
print_error_errno(cmd, "fail to open \"%s\"", fn[i]);
// TODO sam_itr_next() doesn't yet work for SAM files,
// so for those keep the headers around for use with sam_read1()
if (hts_get_format(fp[i])->format == sam) hdr[i] = hin;
- else { bam_hdr_destroy(hin); hdr[i] = NULL; }
+ else { sam_hdr_destroy(hin); hdr[i] = NULL; }
if ((translation_tbl+i)->lost_coord_sort && !by_qname) {
fprintf(stderr, "[bam_merge_core] Order of targets in file %s caused coordinate sort to be lost\n", fn[i]);
}
// Transform the header into standard form
- hout = finish_merged_header(merged_hdr);
+ if (finish_merged_header(merged_hdr) < 0)
+ goto fail;
+
+ hout = merged_hdr->hdr;
if (!hout) return -1; // FIXME: memory leak
// If we're only merging a specified region move our iters to start at that point
if (reg) {
- int tid, beg, end;
- const char *name_lim;
+ int tid;
+ hts_pos_t beg, end;
- rtrans = rtrans_build(n, hout->n_targets, translation_tbl);
+ rtrans = rtrans_build(n, sam_hdr_nref(hout), translation_tbl);
if (!rtrans) goto mem_fail;
- name_lim = hts_parse_reg(reg, &beg, &end);
- if (name_lim) {
- char *name = malloc(name_lim - reg + 1);
- if (!name) goto mem_fail;
- memcpy(name, reg, name_lim - reg);
- name[name_lim - reg] = '\0';
- tid = bam_name2id(hout, name);
- free(name);
- }
- else {
- // not parsable as a region, but possibly a sequence named "foo:a"
- tid = bam_name2id(hout, reg);
- beg = 0;
- end = INT_MAX;
- }
- if (tid < 0) {
- if (name_lim) fprintf(stderr, "[%s] Region \"%s\" specifies an unknown reference name\n", __func__, reg);
- else fprintf(stderr, "[%s] Badly formatted region: \"%s\"\n", __func__, reg);
+ if (!sam_parse_region(hout, reg, &tid, &beg, &end, 0)) {
+ fprintf(stderr, "[%s] Badly formatted region or unknown reference name: \"%s\"\n", __func__, reg);
goto fail;
}
for (i = 0; i < n; ++i) {
- hts_idx_t *idx = sam_index_load(fp[i], fn[i]);
+ hts_idx_t *idx = NULL;
+ // If index filename has not been specfied, look in BAM folder
+ if (fn_idx != NULL) {
+ idx = sam_index_load2(fp[i], fn[i], fn_idx[i]);
+ } else {
+ idx = sam_index_load(fp[i], fn[i]);
+ }
// (rtrans[i*n+tid]) Look up what hout tid translates to in input tid space
- int mapped_tid = rtrans[i*hout->n_targets+tid];
+ int mapped_tid = rtrans[i*sam_hdr_nref(hout)+tid];
if (idx == NULL) {
fprintf(stderr, "[%s] failed to load index for %s. Random alignment retrieval only works for indexed BAM or CRAM files.\n",
__func__, fn[i]);
if (mapped_tid != INT32_MIN) {
fprintf(stderr,
"[%s] failed to get iterator over "
- "{%s, %d, %d, %d}\n",
+ "{%s, %d, %"PRIhts_pos", %"PRIhts_pos"}\n",
__func__, fn[i], mapped_tid, beg, end);
} else {
fprintf(stderr,
res = iter[i] ? sam_itr_next(fp[i], iter[i], h->entry.bam_record) : sam_read1(fp[i], hdr[i], h->entry.bam_record);
if (res >= 0) {
bam_translate(h->entry.bam_record, translation_tbl + i);
- h->pos = ((uint64_t)h->entry.bam_record->core.tid<<32) | (uint32_t)((int32_t)h->entry.bam_record->core.pos+1);
+ h->tid = h->entry.bam_record->core.tid;
+ h->pos = (uint64_t)(h->entry.bam_record->core.pos + 1);
h->rev = bam_is_rev(h->entry.bam_record);
h->idx = idx++;
if (g_is_by_tag) {
print_error_errno(cmd, "failed to create \"%s\"", out);
return -1;
}
+ if (!no_pg && sam_hdr_add_pg(hout, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error(cmd, "failed to add PG line to the header of \"%s\"", out);
+ sam_close(fpout);
+ return -1;
+ }
if (sam_hdr_write(fpout, hout) != 0) {
print_error_errno(cmd, "failed to write header to \"%s\"", out);
sam_close(fpout);
return -1;
}
+ if (write_index) {
+ if (!(out_idx_fn = auto_index(fpout, out, hout))){
+ sam_close(fpout);
+ return -1;
+ }
+ }
if (!(flag & MERGE_UNCOMP)) hts_set_threads(fpout, n_threads);
// Begin the actual merge
if (sam_write1(fpout, hout, b) < 0) {
print_error_errno(cmd, "failed writing to \"%s\"", out);
sam_close(fpout);
+ free(out_idx_fn);
return -1;
}
if ((j = (iter[heap->i]? sam_itr_next(fp[heap->i], iter[heap->i], b) : sam_read1(fp[heap->i], hdr[heap->i], b))) >= 0) {
bam_translate(b, translation_tbl + heap->i);
- heap->pos = ((uint64_t)b->core.tid<<32) | (uint32_t)((int)b->core.pos+1);
+ heap->tid = b->core.tid;
+ heap->pos = (uint64_t)(b->core.pos + 1);
heap->rev = bam_is_rev(b);
heap->idx = idx++;
if (g_is_by_tag) {
ks_heapadjust(heap, 0, n, heap);
}
+ if (write_index) {
+ if (sam_idx_save(fpout) < 0) {
+ print_error_errno("merge", "writing index failed");
+ goto fail;
+ }
+ }
+ free(out_idx_fn);
+
// Clean up and close
if (flag & MERGE_RG) {
for (i = 0; i != n; ++i) free(RG[i]);
for (i = 0; i < n; ++i) {
trans_tbl_destroy(translation_tbl + i);
hts_itr_destroy(iter[i]);
- bam_hdr_destroy(hdr[i]);
+ sam_hdr_destroy(hdr[i]);
sam_close(fp[i]);
}
- bam_hdr_destroy(hin);
- bam_hdr_destroy(hout);
+ sam_hdr_destroy(hin);
+ sam_hdr_destroy(hout);
free_merged_header(merged_hdr);
free(RG); free(translation_tbl); free(fp); free(heap); free(iter); free(hdr);
if (sam_close(fpout) < 0) {
for (i = 0; i < n; ++i) {
if (translation_tbl && translation_tbl[i].tid_trans) trans_tbl_destroy(translation_tbl + i);
if (iter && iter[i]) hts_itr_destroy(iter[i]);
- if (hdr && hdr[i]) bam_hdr_destroy(hdr[i]);
+ if (hdr && hdr[i]) sam_hdr_destroy(hdr[i]);
if (fp && fp[i]) sam_close(fp[i]);
if (heap && heap[i].entry.bam_record) bam_destroy1(heap[i].entry.bam_record);
}
- if (hout) bam_hdr_destroy(hout);
+ if (hout) sam_hdr_destroy(hout);
free(RG);
free(translation_tbl);
free(hdr);
free(heap);
free(fp);
free(rtrans);
+ free(out_idx_fn);
return -1;
}
strcpy(mode, "wb");
if (flag & MERGE_UNCOMP) strcat(mode, "0");
else if (flag & MERGE_LEVEL1) strcat(mode, "1");
- return bam_merge_core2(by_qname, NULL, out, mode, headers, n, fn, flag, reg, 0, "merge", NULL, NULL);
+ return bam_merge_core2(by_qname, NULL, out, mode, headers, n, fn, NULL, flag, reg, 0, "merge", NULL, NULL, 0, NULL, 1);
}
static void merge_usage(FILE *to)
" -c Combine @RG headers with colliding IDs [alter IDs to be distinct]\n"
" -p Combine @PG headers with colliding IDs [alter IDs to be distinct]\n"
" -s VALUE Override random seed\n"
-" -b FILE List of input BAM filenames, one per line [null]\n");
- sam_global_opt_help(to, "-.O..@");
+" -b FILE List of input BAM filenames, one per line [null]\n"
+" -X Use customized index files\n"
+" --no-PG do not add a PG line\n");
+ sam_global_opt_help(to, "-.O..@..");
}
int bam_merge(int argc, char *argv[])
{
- int c, is_by_qname = 0, flag = 0, ret = 0, level = -1;
+ int c, is_by_qname = 0, flag = 0, ret = 0, level = -1, has_index_file = 0;
char *fn_headers = NULL, *reg = NULL, mode[12];
- char *sort_tag = NULL;
+ char *sort_tag = NULL, *arg_list = NULL;
long random_seed = (long)time(NULL);
char** fn = NULL;
- int fn_size = 0;
+ char** fn_idx = NULL;
+ int fn_size = 0, no_pg = 0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 0, '@'),
{ "threads", required_argument, NULL, '@' },
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
return 0;
}
- while ((c = getopt_long(argc, argv, "h:nru1R:f@:l:cps:b:O:t:", lopts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "h:nru1R:f@:l:cps:b:O:t:X", lopts, NULL)) >= 0) {
switch (c) {
case 'r': flag |= MERGE_RG; break;
case 'f': flag |= MERGE_FORCE; break;
- case 'h': fn_headers = strdup(optarg); break;
+ case 'h': fn_headers = optarg; break;
case 'n': is_by_qname = 1; break;
- case 't': sort_tag = strdup(optarg); break;
+ case 't': sort_tag = optarg; break;
case '1': flag |= MERGE_LEVEL1; level = 1; break;
case 'u': flag |= MERGE_UNCOMP; level = 0; break;
case 'R': reg = strdup(optarg); break;
case 'c': flag |= MERGE_COMBINE_RG; break;
case 'p': flag |= MERGE_COMBINE_PG; break;
case 's': random_seed = atol(optarg); break;
+ case 'X': has_index_file = 1; break; // -X flag for index filename
case 'b': {
// load the list of files to read
+ if (has_index_file) {
+ fprintf(stderr,"Error: The -b option cannot be combined with -X\n");
+ ret = 1; goto end;
+ }
int nfiles;
char **fn_read = hts_readlines(optarg, &nfiles);
if (fn_read) {
}
break;
}
-
+ case 1: no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': merge_usage(stderr); return 1;
return 1;
}
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("merge", "failed to create arg_list");
+ return 1;
+ }
+
srand48(random_seed);
if (!(flag & MERGE_FORCE) && strcmp(argv[optind], "-")) {
FILE *fp = fopen(argv[optind], "rb");
}
}
- int nargcfiles = argc - (optind+1);
+ int nargcfiles = 0;
+ if (has_index_file) { // Calculate # of input BAM files
+ if ((argc - optind - 1) % 2 != 0) {
+ fprintf(stderr, "Odd number of filenames detected! Each BAM file should have an index file\n");
+ return 1;
+ }
+ nargcfiles = (argc - optind - 1) / 2;
+ } else {
+ nargcfiles = argc - optind - 1;
+ }
+
if (nargcfiles > 0) {
// Add argc files to end of array
fn = realloc(fn, (fn_size+nargcfiles) * sizeof(char*));
if (fn == NULL) { ret = 1; goto end; }
memcpy(fn+fn_size, argv + (optind+1), nargcfiles * sizeof(char*));
+
+ if(has_index_file) {
+ fn_idx = realloc(fn_idx, nargcfiles * sizeof(char*));
+ if (fn_idx == NULL) { ret = 1; goto end; }
+ memcpy(fn_idx+fn_size, argv + nargcfiles + (optind+1), nargcfiles * sizeof(char*));
+ }
}
if (fn_size+nargcfiles < 1) {
print_error("merge", "You must specify at least one (and usually two or more) input files");
merge_usage(stderr);
+ free(fn_idx);
return 1;
}
strcpy(mode, "wb");
sam_open_mode(mode+1, argv[optind], NULL);
if (level >= 0) sprintf(strchr(mode, '\0'), "%d", level < 9? level : 9);
if (bam_merge_core2(is_by_qname, sort_tag, argv[optind], mode, fn_headers,
- fn_size+nargcfiles, fn, flag, reg, ga.nthreads,
- "merge", &ga.in, &ga.out) < 0)
+ fn_size+nargcfiles, fn, fn_idx, flag, reg, ga.nthreads,
+ "merge", &ga.in, &ga.out, ga.write_index, arg_list, no_pg) < 0)
ret = 1;
end:
for (i=0; i<fn_size; i++) free(fn[i]);
}
free(fn);
+ free(fn_idx);
free(reg);
- free(fn_headers);
+ free(arg_list);
sam_global_args_free(&ga);
return ret;
}
static inline int heap_add_read(heap1_t *heap, int nfiles, samFile **fp,
int num_in_mem, buf_region *in_mem,
- bam1_tag *buf, uint64_t *idx, bam_hdr_t *hout) {
+ bam1_tag *buf, uint64_t *idx, sam_hdr_t *hout) {
int i = heap->i, res;
if (i < nfiles) { // read from file
res = sam_read1(fp[i], hout, heap->entry.bam_record);
}
}
if (res >= 0) {
- heap->pos = (((uint64_t)heap->entry.bam_record->core.tid<<32)
- | (uint32_t)((int32_t)heap->entry.bam_record->core.pos+1));
+ heap->tid = heap->entry.bam_record->core.tid;
+ heap->pos = (uint64_t)(heap->entry.bam_record->core.pos + 1);
heap->rev = bam_is_rev(heap->entry.bam_record);
heap->idx = (*idx)++;
if (g_is_by_tag) {
}
static int bam_merge_simple(int by_qname, char *sort_tag, const char *out,
- const char *mode, bam_hdr_t *hout,
+ const char *mode, sam_hdr_t *hout,
int n, char * const *fn, int num_in_mem,
buf_region *in_mem, bam1_tag *buf, int n_threads,
const char *cmd, const htsFormat *in_fmt,
- const htsFormat *out_fmt) {
+ const htsFormat *out_fmt, char *arg_list, int no_pg,
+ int write_index) {
samFile *fpout = NULL, **fp = NULL;
heap1_t *heap = NULL;
uint64_t idx = 0;
int i, heap_size = n + num_in_mem;
+ char *out_idx_fn = NULL;
g_is_by_qname = by_qname;
if (sort_tag) {
g_is_by_tag = 1;
g_sort_tag[0] = sort_tag[0];
- g_sort_tag[1] = sort_tag[1];
+ g_sort_tag[1] = sort_tag[0] ? sort_tag[1] : '\0';
}
if (n > 0) {
fp = (samFile**)calloc(n, sizeof(samFile*));
// Open each file, read the header and put the first read into the heap
for (i = 0; i < heap_size; i++) {
- bam_hdr_t *hin;
+ sam_hdr_t *hin;
heap1_t *h = &heap[i];
if (i < n) {
goto fail;
}
// ... and throw it away as we don't really need it
- bam_hdr_destroy(hin);
+ sam_hdr_destroy(hin);
}
// Get a read into the heap
return -1;
}
+ if (!no_pg && sam_hdr_add_pg(hout, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error(cmd, "failed to add PG line to the header of \"%s\"", out);
+ sam_close(fpout);
+ return -1;
+ }
+
if (n_threads > 1) hts_set_threads(fpout, n_threads);
if (sam_hdr_write(fpout, hout) != 0) {
return -1;
}
+ if (write_index) {
+ if (!(out_idx_fn = auto_index(fpout, out, hout))){
+ sam_close(fpout);
+ return -1;
+ }
+ }
+
// Now do the merge
ks_heapmake(heap, heap_size, heap);
while (heap->pos != HEAP_EMPTY) {
bam1_t *b = heap->entry.bam_record;
if (sam_write1(fpout, hout, b) < 0) {
print_error_errno(cmd, "failed writing to \"%s\"", out);
- sam_close(fpout);
- return -1;
+ goto fail;
}
if (heap_add_read(heap, n, fp, num_in_mem, in_mem, buf, &idx, hout) < 0) {
assert(heap->i < n);
}
free(fp);
free(heap);
+
+ if (write_index) {
+ if (sam_idx_save(fpout) < 0) {
+ print_error_errno("merge", "writing index failed");
+ goto fail;
+ }
+ free(out_idx_fn);
+ }
+
if (sam_close(fpout) < 0) {
print_error(cmd, "error closing output file");
return -1;
fail:
for (i = 0; i < n; i++) {
if (fp && fp[i]) sam_close(fp[i]);
- if (heap && heap[i].entry.bam_record) bam_destroy1(heap[i].entry.bam_record);
+ }
+ for (i = 0; i < heap_size; i++) {
+ if (heap && heap[i].i < n && heap[i].entry.bam_record)
+ bam_destroy1(heap[i].entry.bam_record);
}
free(fp);
free(heap);
if (fpout) sam_close(fpout);
+ free(out_idx_fn);
return -1;
}
if (t != 0) return t;
return (int) (a.bam_record->core.flag&0xc0) - (int) (b.bam_record->core.flag&0xc0);
} else {
- pa = (uint64_t)a.bam_record->core.tid<<32|(a.bam_record->core.pos+1);
- pb = (uint64_t)b.bam_record->core.tid<<32|(b.bam_record->core.pos+1);
+ pa = a.bam_record->core.tid;
+ pb = b.bam_record->core.tid;
+
+ if (pa == pb) {
+ pa = (uint64_t)(a.bam_record->core.pos+1);
+ pb = (uint64_t)(b.bam_record->core.pos+1);
+ }
if (pa == pb) {
pa = bam_is_rev(a.bam_record);
size_t buf_len;
const char *prefix;
bam1_tag *buf;
- const bam_hdr_t *h;
+ const sam_hdr_t *h;
int index;
int error;
int no_save;
// Returns 0 for success
// -1 for failure
-static int write_buffer(const char *fn, const char *mode, size_t l, bam1_tag *buf, const bam_hdr_t *h, int n_threads, const htsFormat *fmt)
+static int write_buffer(const char *fn, const char *mode, size_t l, bam1_tag *buf,
+ const sam_hdr_t *h, int n_threads, const htsFormat *fmt,
+ char *arg_list, int no_pg, int write_index)
{
size_t i;
samFile* fp;
+ char *out_idx_fn = NULL;
+
fp = sam_open_format(fn, mode, fmt);
if (fp == NULL) return -1;
- if (sam_hdr_write(fp, h) != 0) goto fail;
+ if (!no_pg && sam_hdr_add_pg((sam_hdr_t *)h, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ goto fail;
+ }
+ if (sam_hdr_write(fp, (sam_hdr_t *)h) != 0) goto fail;
+
+ if (write_index) {
+ if (!(out_idx_fn = auto_index(fp, fn, (sam_hdr_t *)h))) goto fail;
+ }
+
if (n_threads > 1) hts_set_threads(fp, n_threads);
for (i = 0; i < l; ++i) {
- if (sam_write1(fp, h, buf[i].bam_record) < 0) goto fail;
+ if (sam_write1(fp, (sam_hdr_t *)h, buf[i].bam_record) < 0) goto fail;
}
+
+ if (write_index) {
+ if (sam_idx_save(fp) < 0) {
+ print_error_errno("merge", "writing index failed");
+ goto fail;
+ }
+ free(out_idx_fn);
+ }
+
+
if (sam_close(fp) < 0) return -1;
return 0;
fail:
sam_close(fp);
+ free(out_idx_fn);
return -1;
}
#define NUMBASE 256
-#define STEP 8
-static int ks_radixsort(size_t n, bam1_tag *buf, const bam_hdr_t *h)
+static int ks_radixsort(size_t n, bam1_tag *buf, const sam_hdr_t *h)
{
int curr = 0, ret = -1;
ssize_t i;
bam1_tag *buf_ar2[2], *bam_a, *bam_b;
- uint64_t max_pos = 0, max_digit = 0, shift = 0;
-
+ uint64_t max_pos = 1;
+ uint32_t max_tid = 1, tid_bytes = 0, pos_bytes = 0, byte = 0;
+ uint32_t tid_shift_l, tid_shift_r;
+ int nref = sam_hdr_nref(h);
+
+ // Count number of bytes needed for biggest tid and pos
+ // Notes: Add 1 to core.pos so always positive.
+ // Convert unmapped tid (-1) to number of references so unmapped
+ // sort to the end.
for (i = 0; i < n; i++) {
bam1_t *b = buf[i].bam_record;
- int32_t tid = b->core.tid == -1 ? h->n_targets : b->core.tid;
- buf[i].u.pos = (uint64_t)tid<<32 | (b->core.pos+1)<<1 | bam_is_rev(b);
- if (max_pos < buf[i].u.pos)
- max_pos = buf[i].u.pos;
+ uint32_t tid = b->core.tid == -1 ? nref : b->core.tid;
+ uint64_t pos = ((uint64_t)(b->core.pos + 1) << 1) | bam_is_rev(b);
+ if (max_tid < tid)
+ max_tid = tid;
+ if (max_pos < pos)
+ max_pos = pos;
}
- while (max_pos) {
- ++max_digit;
- max_pos = max_pos >> 1;
+ for (; max_pos > 0; max_pos >>= 8) pos_bytes++;
+ for (; max_tid > 0; max_tid >>= 8) tid_bytes++;
+ assert(pos_bytes + tid_bytes < sizeof(buf[0].u.pos_tid));
+
+ tid_shift_l = pos_bytes * 8;
+ tid_shift_r = 64 - tid_shift_l;
+
+ // Write position and tid into bam1_tag::u::pos_tid using minimum number
+ // of bytes required. Values are stored little-endian so that we
+ // get a least-significant digit (byte) radix sort.
+ for (i = 0; i < n; i++) {
+ bam1_t *b = buf[i].bam_record;
+ uint32_t tid = b->core.tid == -1 ? nref : b->core.tid;
+ // 'pos' here includes as many bytes of tid as will fit
+ // in the space remaining above pos_bytes. The rest of tid
+ // is written out separately.
+ uint64_t pos = (bam_is_rev(b) |
+ ((uint64_t)(b->core.pos + 1) << 1) |
+ (tid_shift_l < 64 ? (uint64_t) tid << tid_shift_l : 0));
+ u64_to_le(pos, buf[i].u.pos_tid);
+ u32_to_le(tid_shift_r < 32 ? tid >> tid_shift_r : 0,
+ &buf[i].u.pos_tid[8]);
}
buf_ar2[0] = buf;
goto err;
}
- while (shift < max_digit){
+ // Least-significant digit radix sort (where "digits" are bytes)
+ for (byte = 0; byte < pos_bytes + tid_bytes; byte++) {
size_t remainders[NUMBASE] = { 0 };
bam_a = buf_ar2[curr]; bam_b = buf_ar2[1-curr];
for (i = 0; i < n; ++i)
- remainders[(bam_a[i].u.pos >> shift) % NUMBASE]++;
+ remainders[bam_a[i].u.pos_tid[byte]]++;
for (i = 1; i < NUMBASE; ++i)
remainders[i] += remainders[i - 1];
for (i = n - 1; i >= 0; i--) {
- size_t j = --remainders[(bam_a[i].u.pos >> shift) % NUMBASE];
+ size_t j = --remainders[bam_a[i].u.pos_tid[byte]];
bam_b[j] = bam_a[i];
}
- shift += STEP;
curr = 1 - curr;
}
if (curr == 1) {
return 0;
}
- if (write_buffer(name, "wcx1", w->buf_len, w->buf, w->h, 0, &fmt) < 0)
+ if (write_buffer(name, "wcx1", w->buf_len, w->buf, w->h, 0, &fmt, NULL, 1, 0) < 0)
w->error = errno;
} else {
- if (write_buffer(name, "wbx1", w->buf_len, w->buf, w->h, 0, NULL) < 0)
+ if (write_buffer(name, "wbx1", w->buf_len, w->buf, w->h, 0, NULL, NULL, 1, 0) < 0)
w->error = errno;
}
}
static int sort_blocks(int n_files, size_t k, bam1_tag *buf, const char *prefix,
- const bam_hdr_t *h, int n_threads, buf_region *in_mem)
+ const sam_hdr_t *h, int n_threads, buf_region *in_mem)
{
int i;
size_t pos, rest;
@param max_mem approxiate maximum memory (very inaccurate)
@param in_fmt input file format options
@param out_fmt output file format and options
+ @param arg_list command string for PG line
+ @param no_pg if 1, do not add a new PG line
+ @paran write_index create index for the output file
@return 0 for successful sorting, negative on errors
@discussion It may create multiple temporary subalignment files
int bam_sort_core_ext(int is_by_qname, char* sort_by_tag, const char *fn, const char *prefix,
const char *fnout, const char *modeout,
size_t _max_mem, int n_threads,
- const htsFormat *in_fmt, const htsFormat *out_fmt)
+ const htsFormat *in_fmt, const htsFormat *out_fmt,
+ char *arg_list, int no_pg, int write_index)
{
int ret = -1, res, i, n_files = 0;
size_t max_k, k, max_mem, bam_mem_offset;
- bam_hdr_t *header = NULL;
+ sam_hdr_t *header = NULL;
samFile *fp;
bam1_tag *buf = NULL;
bam1_t *b = bam_init1();
g_is_by_qname = is_by_qname;
if (sort_by_tag) {
g_is_by_tag = 1;
- strncpy(g_sort_tag, sort_by_tag, 2);
+ g_sort_tag[0] = sort_by_tag[0];
+ g_sort_tag[1] = sort_by_tag[0] ? sort_by_tag[1] : '\0';
}
max_mem = _max_mem * n_threads;
else
new_so = "coordinate";
- if (sam_hdr_change_HD(header, "SO", new_so) != 0) {
- print_error("sort",
- "failed to change sort order header to '%s'\n", new_so);
+ if ((-1 == sam_hdr_update_hd(header, "SO", new_so))
+ && (-1 == sam_hdr_add_line(header, "HD", "VN", SAM_FORMAT_VERSION, "SO", new_so, NULL))
+ ) {
+ print_error("sort", "failed to change sort order header to '%s'\n", new_so);
goto err;
}
- if (sam_hdr_change_HD(header, "GO", NULL) != 0) {
- print_error("sort",
- "failed to delete group order header\n");
+
+ if (-1 == sam_hdr_remove_tag_hd(header, "GO")) {
+ print_error("sort", "failed to delete group order header\n");
goto err;
}
// write the final output
if (n_files == 0 && num_in_mem < 2) { // a single block
- if (write_buffer(fnout, modeout, k, buf, header, n_threads, out_fmt) != 0) {
+ if (write_buffer(fnout, modeout, k, buf, header, n_threads, out_fmt, arg_list, no_pg, write_index) != 0) {
print_error_errno("sort", "failed to create \"%s\"", fnout);
goto err;
}
}
if (bam_merge_simple(is_by_qname, sort_by_tag, fnout, modeout, header,
n_files, fns, num_in_mem, in_mem, buf,
- n_threads, "sort", in_fmt, out_fmt) < 0) {
+ n_threads, "sort", in_fmt, out_fmt, arg_list,
+ no_pg, write_index) < 0) {
// Propagate bam_merge_simple() failure; it has already emitted a
// message explaining the failure, so no further message is needed.
goto err;
free(buf);
free(bam_mem);
free(in_mem);
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
if (fp) sam_close(fp);
return ret;
}
char *fnout = calloc(strlen(prefix) + 4 + 1, 1);
if (!fnout) return -1;
sprintf(fnout, "%s.bam", prefix);
- ret = bam_sort_core_ext(is_by_qname, NULL, fn, prefix, fnout, "wb", max_mem, 0, NULL, NULL);
+ ret = bam_sort_core_ext(is_by_qname, NULL, fn, prefix, fnout, "wb", max_mem, 0, NULL, NULL, NULL, 1, 0);
free(fnout);
return ret;
}
" -n Sort by read name\n"
" -t TAG Sort by value of TAG. Uses position as secondary index (or read name if -n is set)\n"
" -o FILE Write final output to FILE rather than standard output\n"
-" -T PREFIX Write temporary files to PREFIX.nnnn.bam\n");
- sam_global_opt_help(fp, "-.O..@");
+" -T PREFIX Write temporary files to PREFIX.nnnn.bam\n"
+" --no-PG do not add a PG line\n");
+ sam_global_opt_help(fp, "-.O..@-.");
}
static void complain_about_memory_setting(size_t max_mem) {
int bam_sort(int argc, char *argv[])
{
size_t max_mem = SORT_DEFAULT_MEGS_PER_THREAD << 20;
- int c, nargs, is_by_qname = 0, ret, o_seen = 0, level = -1;
- char* sort_tag = NULL;
+ int c, nargs, is_by_qname = 0, ret, o_seen = 0, level = -1, no_pg = 0;
+ char* sort_tag = NULL, *arg_list = NULL;
char *fnout = "-", modeout[12];
kstring_t tmpprefix = { 0, 0, NULL };
struct stat st;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 0, '@'),
{ "threads", required_argument, NULL, '@' },
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
switch (c) {
case 'o': fnout = optarg; o_seen = 1; break;
case 'n': is_by_qname = 1; break;
- case 't': sort_tag = strdup(optarg); break;
+ case 't': sort_tag = optarg; break;
case 'm': {
char *q;
max_mem = strtol(optarg, &q, 0);
}
case 'T': kputs(optarg, &tmpprefix); break;
case 'l': level = atoi(optarg); break;
+ case 1: no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
goto sort_end;
}
+ if (ga.write_index && (is_by_qname || sort_tag)) {
+ fprintf(stderr, "[W::bam_sort] Ignoring --write-index as it only works for position sorted files.\n");
+ ga.write_index = 0;
+ }
+
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("sort", "failed to create arg_list");
+ return 1;
+ }
+
if (max_mem < (SORT_MIN_MEGS_PER_THREAD << 20)) {
complain_about_memory_setting(max_mem);
ret = EXIT_FAILURE;
ret = bam_sort_core_ext(is_by_qname, sort_tag, (nargs > 0)? argv[optind] : "-",
tmpprefix.s, fnout, modeout, max_mem, ga.nthreads,
- &ga.in, &ga.out);
+ &ga.in, &ga.out, arg_list, no_pg, ga.write_index);
if (ret >= 0)
ret = EXIT_SUCCESS;
else {
sort_end:
free(tmpprefix.s);
+ free(arg_list);
sam_global_args_free(&ga);
return ret;
/* bam_sort.c -- sorting and merging.
- Copyright (C) 2008-2016 Genome Research Ltd.
+ Copyright (C) 2008-2019 Genome Research Ltd.
Portions copyright (C) 2009-2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include "htslib/klist.h"
#include "htslib/kstring.h"
#include "htslib/sam.h"
+#include "htslib/hts_endian.h"
#include "sam_opts.h"
#include "samtools.h"
bam1_t *bam_record;
union {
const uint8_t *tag;
- uint64_t pos;
+ uint8_t pos_tid[12];
} u;
} bam1_tag;
return *pa? 1 : *pb? -1 : 0;
}
-#define HEAP_EMPTY UINT64_MAX
+#define HEAP_EMPTY (UINT64_MAX >> 1)
typedef struct {
int i;
- uint32_t rev;
- uint64_t pos, idx;
+ uint32_t tid;
+ uint64_t pos:63, rev:1, idx;
bam1_tag entry;
} heap1_t;
fb = b.entry.bam_record->core.flag & 0xc0;
if (fa != fb) return fa > fb;
} else {
+ if (a.tid != b.tid) return a.tid > b.tid;
if (a.pos != b.pos) return a.pos > b.pos;
if (a.rev != b.rev) return a.rev > b.rev;
}
KSORT_INIT(heap, heap1_t, heap_lt)
typedef struct merged_header {
- kstring_t out_hd;
- kstring_t out_sq;
+ sam_hdr_t *hdr;
kstring_t out_rg;
kstring_t out_pg;
kstring_t out_co;
bool lost_coord_sort;
} trans_tbl_t;
-/* Something to look like a regmatch_t */
-typedef struct hdr_match {
- ptrdiff_t rm_so;
- ptrdiff_t rm_eo;
-} hdr_match_t;
-
-/*
- * Search for header lines of a particular record type.
- *
- * This replaces a regex search for something like /^@SQ.*\tSN:([^\t]+).*$/
- * but is much quicker. The locations found are returned in *matches,
- * which has a signature the same as that of a regmatch_t.
- *
- * rec is the record type to match (i.e. @HD, @SQ, @PG or @RG)
- * tag is a tag type in the record to match (SN for @SQ, ID for @PG or @RG)
- *
- * The location of the record (if found) is returned in matches[0]
- * If tag is not NULL, the record is searched for the presence of the
- * given tag. If found, the location of the value is returned in matches[1].
- * If the tag isn't found then the record is ignored and the search resumes
- * on the next header line.
- *
- * For simplicity, some assumptions are made about rec and tag:
- * rec should include the leading '@' sign and be three characters long.
- * tag should be exactly two characters long.
- * These are always string constants when this is called below, so we don't
- * bother to check here.
- *
- * Returns 0 if a match was found, -1 if not.
- */
-
-
-static int hdr_line_match(const char *text, const char *rec,
- const char *tag, hdr_match_t *matches) {
- const char *line_start, *line_end = text;
- const char *tag_start, *tag_end;
-
- for (;;) {
- // Find record, ensure either at start of text or follows '\n'
- line_start = strstr(line_end, rec);
- while (line_start && line_start > text && *(line_start - 1) != '\n') {
- line_start = strstr(line_start + 3, rec);
- }
- if (!line_start) return -1;
-
- // Find end of header line
- line_end = strchr(line_start, '\n');
- if (!line_end) line_end = line_start + strlen(line_start);
-
- matches[0].rm_so = line_start - text;
- matches[0].rm_eo = line_end - text;
- if (!tag) return 0; // Match found if not looking for tag.
-
- for (tag_start = line_start + 3; tag_start < line_end; tag_start++) {
- // Find possible tag start. Hacky but quick.
- while (*tag_start > '\n') tag_start++;
-
- // Check it
- if (tag_start[0] == '\t'
- && strncmp(tag_start + 1, tag, 2) == 0
- && tag_start[3] == ':') {
- // Found tag, record location and return.
- tag_end = tag_start + 4;
- while (*tag_end && *tag_end != '\t' && *tag_end != '\n')
- ++tag_end;
- matches[1].rm_so = tag_start - text + 4;
- matches[1].rm_eo = tag_end - text;
- return 0;
- }
- }
- // Couldn't find tag, try again from end of current record.
- }
-}
-
static void trans_tbl_destroy(trans_tbl_t *tbl) {
khiter_t iter;
merged_hdr = calloc(1, sizeof(*merged_hdr));
if (merged_hdr == NULL) return NULL;
+ merged_hdr->hdr = sam_hdr_init();
+ if (!merged_hdr->hdr) goto fail;
+
merged_hdr->targets_sz = 16;
merged_hdr->target_name = malloc(merged_hdr->targets_sz
* sizeof(*merged_hdr->target_name));
kh_destroy(c2i, merged_hdr->sq_tids);
free(merged_hdr->target_name);
free(merged_hdr->target_len);
+ sam_hdr_destroy(merged_hdr->hdr);
free(merged_hdr);
return NULL;
}
return kputsn(src + from, to - from, dest) != to - from;
}
-// Append a header line match to kstring
-static inline int match_to_ks(const char *src, const hdr_match_t *match,
- kstring_t *dest) {
- return range_to_ks(src, match->rm_so, match->rm_eo, dest);
-}
-
// Append a kstring to a kstring
static inline int ks_to_ks(kstring_t *src, kstring_t *dest) {
return kputsn(ks_str(src), ks_len(src), dest) != ks_len(src);
*/
static int trans_tbl_add_hd(merged_header_t* merged_hdr,
- bam_hdr_t *translate) {
- hdr_match_t match = {0, 0};
+ sam_hdr_t *translate) {
+ kstring_t hd_line = { 0, 0, NULL };
+ int res;
// TODO: handle case when @HD needs merging.
if (merged_hdr->have_hd) return 0;
- if (hdr_line_match(translate->text, "@HD", NULL, &match) != 0) {
- return 0;
+ res = sam_hdr_find_hd(translate, &hd_line);
+ if (res < -1) {
+ print_error("merge", "failed to get @HD line from header");
+ return -1;
}
- if (match_to_ks(translate->text, &match, &merged_hdr->out_hd)) goto memfail;
- if (kputc('\n', &merged_hdr->out_hd) == EOF) goto memfail;
- merged_hdr->have_hd = true;
-
- return 0;
-
- memfail:
- perror(__func__);
- return -1;
-}
-
-static inline int grow_target_list(merged_header_t* merged_hdr) {
- size_t new_size;
- char **new_names;
- uint32_t *new_len;
-
- new_size = merged_hdr->targets_sz * 2;
- new_names = realloc(merged_hdr->target_name, sizeof(*new_names) * new_size);
- if (!new_names) goto fail;
- merged_hdr->target_name = new_names;
+ if (res < 0) // Not found
+ return 0;
- new_len = realloc(merged_hdr->target_len, sizeof(*new_len) * new_size);
- if (!new_len) goto fail;
- merged_hdr->target_len = new_len;
+ if (sam_hdr_add_lines(merged_hdr->hdr, hd_line.s, hd_line.l) < 0) {
+ print_error("merge", "failed to add @HD line to new header");
+ free(hd_line.s);
+ return -1;
+ }
- merged_hdr->targets_sz = new_size;
+ free(hd_line.s);
+ merged_hdr->have_hd = true;
return 0;
-
- fail:
- perror(__func__);
- return -1;
}
/*
* Returns 0 on success, -1 on failure.
*/
-static int trans_tbl_add_sq(merged_header_t* merged_hdr, bam_hdr_t *translate,
+static int trans_tbl_add_sq(merged_header_t* merged_hdr, sam_hdr_t *translate,
trans_tbl_t* tbl) {
-
- kstring_t *out_text = &merged_hdr->out_sq;
- khash_t(c2i)* sq_tids = merged_hdr->sq_tids;
- hdr_match_t *new_sq_matches = NULL;
- char *text;
- hdr_match_t matches[2];
int32_t i;
- int32_t old_n_targets = merged_hdr->n_targets;
- khiter_t iter;
- int min_tid = -1;
+ int min_tid = -1, res;
+ kstring_t sq_line = { 0, 0, NULL }, sq_sn = { 0, 0, NULL };
// Fill in the tid part of the translation table, adding new targets
// to the merged header as we go.
- for (i = 0; i < translate->n_targets; ++i) {
+ for (i = 0; i < sam_hdr_nref(translate); ++i) {
+ int trans_tid;
+ sq_sn.l = 0;
+ res = sam_hdr_find_tag_pos(translate, "SQ", i, "SN", &sq_sn);
+ if (res < 0) {
+ print_error("merge", "failed to get @SQ SN #%d from header", i + 1);
+ goto fail;
+ }
- // Check if it's a new target.
- iter = kh_get(c2i, sq_tids, translate->target_name[i]);
+ trans_tid = sam_hdr_name2tid(merged_hdr->hdr, sq_sn.s);
+ if (trans_tid < -1) {
+ print_error("merge", "failed to lookup ref");
+ goto fail;
+ }
- if (iter == kh_end(sq_tids)) {
- int ret;
+ if (trans_tid < 0) {
// Append missing entries to out_hdr
-
- if (merged_hdr->n_targets == merged_hdr->targets_sz) {
- if (grow_target_list(merged_hdr)) goto fail;
+ sq_line.l = 0;
+ res = sam_hdr_find_line_id(translate, "SQ", "SN", sq_sn.s, &sq_line);
+ if (res < 0) {
+ print_error("merge", "failed to get @SQ SN:%s from header", sq_sn.s);
+ goto fail;
}
- merged_hdr->target_name[merged_hdr->n_targets] = strdup(translate->target_name[i]);
- if (merged_hdr->target_name[merged_hdr->n_targets] == NULL) goto memfail;
- merged_hdr->target_len[merged_hdr->n_targets] = translate->target_len[i];
+ trans_tid = sam_hdr_nref(merged_hdr->hdr);
- // Record the new identifier for reference below,
- // and when building the ttable for other inputs.
- iter = kh_put(c2i, sq_tids,
- merged_hdr->target_name[merged_hdr->n_targets], &ret);
- if (ret < 0) {
- free(merged_hdr->target_name[merged_hdr->n_targets]);
- goto memfail;
+ res = sam_hdr_add_lines(merged_hdr->hdr, sq_line.s, sq_line.l);
+ if (res < 0) {
+ print_error("merge", "failed to add @SQ SN:%s to new header", sq_sn.s);
+ goto fail;
}
- assert(ret > 0); // Should not be in hash already.
-
- kh_value(sq_tids, iter) = merged_hdr->n_targets;
- tbl->tid_trans[i] = merged_hdr->n_targets++;
- } else {
- tbl->tid_trans[i] = kh_value(sq_tids, iter);
}
+ tbl->tid_trans[i] = trans_tid;
if (tbl->tid_trans[i] > min_tid) {
min_tid = tbl->tid_trans[i];
}
}
- if (merged_hdr->n_targets == old_n_targets)
- return 0; // Everything done if no new targets.
-
- // Otherwise, find @SQ lines in translate->text for all newly added targets.
-
- new_sq_matches = malloc((merged_hdr->n_targets - old_n_targets)
- * sizeof(*new_sq_matches));
- if (new_sq_matches == NULL) goto memfail;
-
- for (i = 0; i < merged_hdr->n_targets - old_n_targets; i++) {
- new_sq_matches[i].rm_so = new_sq_matches[i].rm_eo = -1;
- }
-
- text = translate->text;
- while (hdr_line_match(text, "@SQ", "SN", matches) == 0) {
- // matches[0] is whole line, matches[1] is SN value.
-
- // This is a bit disgusting, but avoids a copy...
- char c = text[matches[1].rm_eo];
- int idx;
-
- text[matches[1].rm_eo] = '\0';
-
- // Look up the SN value in the sq_tids hash.
- iter = kh_get(c2i, sq_tids, text + matches[1].rm_so);
- text[matches[1].rm_eo] = c; // restore text
-
- if (iter == kh_end(sq_tids)) {
- // Warn about this, but it's not really fatal.
- fprintf(samtools_stderr, "[W::%s] @SQ SN (%.*s) found in text header but not binary header.\n",
- __func__,
- (int) (matches[1].rm_eo - matches[1].rm_so),
- text + matches[1].rm_so);
- text += matches[0].rm_eo;
- continue; // Skip to next
- }
-
- idx = kh_value(sq_tids, iter);
- if (idx >= old_n_targets) {
- // is a new SQ, so record position so we can add it to out_text.
- assert(idx < merged_hdr->n_targets);
- ptrdiff_t off = text - translate->text;
- new_sq_matches[idx - old_n_targets].rm_so = matches[0].rm_so + off;
- new_sq_matches[idx - old_n_targets].rm_eo = matches[0].rm_eo + off;
- }
-
- // Carry on searching from end of current match
- text += matches[0].rm_eo;
- }
-
- // Copy the @SQ headers found and recreate any missing from binary header.
- for (i = 0; i < merged_hdr->n_targets - old_n_targets; i++) {
- if (new_sq_matches[i].rm_so >= 0) {
- if (match_to_ks(translate->text, &new_sq_matches[i], out_text))
- goto memfail;
- if (kputc('\n', out_text) == EOF) goto memfail;
- } else {
- if (kputs("@SQ\tSN:", out_text) == EOF ||
- kputs(merged_hdr->target_name[i + old_n_targets], out_text) == EOF ||
- kputs("\tLN:", out_text) == EOF ||
- kputuw(merged_hdr->target_len[i + old_n_targets], out_text) == EOF ||
- kputc('\n', out_text) == EOF) goto memfail;
- }
- }
+ free(sq_line.s);
+ free(sq_sn.s);
- free(new_sq_matches);
return 0;
- memfail:
- perror(__func__);
fail:
- free(new_sq_matches);
+ free(sq_line.s);
+ free(sq_sn.s);
return -1;
}
*
*/
-static klist_t(hdrln) * trans_rg_pg(bool is_rg, bam_hdr_t *translate,
+static klist_t(hdrln) * trans_rg_pg(bool is_rg, sam_hdr_t *translate,
bool merge, khash_t(cset)* known_ids,
khash_t(c2c)* id_map, char *override) {
- hdr_match_t matches[2];
khiter_t iter;
- const char *text = translate->text;
- const char *rec_type = is_rg ? "@RG" : "@PG";
+ int num_ids, i;
+ const char *rec_type = is_rg ? "RG" : "PG";
klist_t(hdrln) *hdr_lines;
hdr_lines = kl_init(hdrln);
// Search through translate's header
- while (hdr_line_match(text, rec_type, "ID", matches) == 0) {
- // matches[0] is the whole @RG/PG line; matches[1] is the ID field value
+ num_ids = sam_hdr_count_lines(translate, rec_type);
+ if (num_ids < 0)
+ goto fail;
+ for (i = 0; i < num_ids; i++) {
kstring_t orig_id = { 0, 0, NULL }; // ID in original header
kstring_t transformed_id = { 0, 0, NULL }; // ID in output header
char *map_value; // Value to store in id_map
bool id_changed; // Have we changed the ID?
bool not_found_in_output; // ID isn't in the output header (yet)
- // Take a copy of the ID as we'll need it for a hash key.
- if (match_to_ks(text, &matches[1], &orig_id)) goto memfail;
+ if (sam_hdr_find_tag_pos(translate, rec_type, i, "ID", &orig_id) < 0)
+ goto fail;
// is our matched ID in our output ID set already?
iter = kh_get(cset, known_ids, ks_str(&orig_id));
// Does this line need to go into our output header?
if (not_found_in_output) {
-
// Take matched line and replace ID with transformed_id
kstring_t new_hdr_line = { 0, 0, NULL };
+ if (sam_hdr_find_line_id(translate, rec_type,
+ "ID", ks_str(&orig_id), &new_hdr_line) < 0){
+ goto fail;
+ }
- if (!id_changed) { // Can just copy
- if (match_to_ks(text, &matches[0], &new_hdr_line)) goto memfail;
- } else { // Substitute new name for original
- if (range_to_ks(text, matches[0].rm_so, matches[1].rm_so,
- &new_hdr_line)) goto memfail;
- if (ks_to_ks(&transformed_id, &new_hdr_line)) goto memfail;
- if (range_to_ks(text, matches[1].rm_eo, matches[0].rm_eo,
- &new_hdr_line)) goto memfail;
+ if (id_changed) {
+ char *idp = strstr(ks_str(&new_hdr_line), "\tID:"), *id_end;
+ ptrdiff_t id_offset, id_len;
+ if (!idp) {
+ print_error("merge", "failed to find ID in \"%s\"\n",
+ ks_str(&new_hdr_line));
+ goto fail;
+ }
+ idp += 4;
+ for (id_end = idp; *id_end >= '\n'; id_end++) {}
+
+ id_offset = idp - new_hdr_line.s;
+ id_len = id_end - idp;
+
+ if (id_len < transformed_id.l) {
+ if (ks_resize(&new_hdr_line, new_hdr_line.l + transformed_id.l - id_len))
+ goto fail;
+ }
+ if (id_len != transformed_id.l) {
+ memmove(new_hdr_line.s + id_offset + transformed_id.l,
+ new_hdr_line.s + id_offset + id_len,
+ new_hdr_line.l - id_offset - id_len + 1);
+ }
+ memcpy(new_hdr_line.s + id_offset, transformed_id.s,
+ transformed_id.l);
}
// append line to output linked list
int in_there = 0;
iter = kh_put(c2c, id_map, ks_release(&orig_id), &in_there);
kh_value(id_map, iter) = map_value;
-
- text += matches[0].rm_eo; // next!
}
// If there are no RG lines in the file and we are overriding add one
memfail:
perror(__func__);
+ fail:
if (hdr_lines) kl_destroy(hdrln, hdr_lines);
return NULL;
}
* Returns 0 on success, -1 on failure.
*/
-static int trans_tbl_init(merged_header_t* merged_hdr, bam_hdr_t* translate,
+static int trans_tbl_init(merged_header_t* merged_hdr, sam_hdr_t* translate,
trans_tbl_t* tbl, bool merge_rg, bool merge_pg,
bool copy_co, char* rg_override)
{
+ kstring_t lines = { 0, 0, NULL };
klist_t(hdrln) *rg_list = NULL;
klist_t(hdrln) *pg_list = NULL;
- tbl->n_targets = translate->n_targets;
+ tbl->n_targets = sam_hdr_nref(translate);
tbl->rg_trans = tbl->pg_trans = NULL;
- tbl->tid_trans = (int*)calloc(translate->n_targets, sizeof(int));
+ tbl->tid_trans = (int*)calloc(tbl->n_targets ? tbl->n_targets : 1,
+ sizeof(int));
if (tbl->tid_trans == NULL) goto memfail;
tbl->rg_trans = kh_init(c2c);
if (tbl->rg_trans == NULL) goto memfail;
goto fail;
// Fix-up PP: tags in the new @PG records and add to output
+ lines.l = 0;
if (finish_rg_pg(false, pg_list, tbl->pg_trans, &merged_hdr->out_pg))
goto fail;
if (copy_co) {
// Just append @CO headers without translation
- const char *line, *end_pointer;
- for (line = translate->text; *line; line = end_pointer + 1) {
- end_pointer = strchr(line, '\n');
- if (strncmp(line, "@CO", 3) == 0) {
- if (end_pointer) {
- if (kputsn(line, end_pointer - line + 1, &merged_hdr->out_co) == EOF)
- goto memfail;
- } else { // Last line with no trailing '\n'
- if (kputs(line, &merged_hdr->out_co) == EOF) goto memfail;
- if (kputc('\n', &merged_hdr->out_co) == EOF) goto memfail;
- }
- }
- if (end_pointer == NULL) break;
+ int num_co = sam_hdr_count_lines(translate, "CO"), i;
+ if (num_co < 0)
+ goto fail;
+
+ for (i = 0; i < num_co; i++) {
+ if (sam_hdr_find_line_pos(translate, "CO", i, &lines) < 0)
+ goto fail;
+ if (ks_to_ks(&lines, &merged_hdr->out_co))
+ goto fail;
+ if (kputc('\n', &merged_hdr->out_co) < 0)
+ goto fail;
}
}
+ free(lines.s);
+
return 0;
memfail:
trans_tbl_destroy(tbl);
if (rg_list) kl_destroy(hdrln, rg_list);
if (pg_list) kl_destroy(hdrln, pg_list);
+ free(lines.s);
return -1;
}
-static inline void move_kstr_to_text(char **text, kstring_t *ks) {
- memcpy(*text, ks_str(ks), ks_len(ks));
- *text += ks_len(ks);
- **text = '\0';
- free(ks_release(ks));
-}
-
-/*
- * Populate a bam_hdr_t struct from data in a merged_header_t.
- */
-
-static bam_hdr_t * finish_merged_header(merged_header_t *merged_hdr) {
- size_t txt_sz;
- char *text;
- bam_hdr_t *hdr;
-
- // Check output text size
- txt_sz = (ks_len(&merged_hdr->out_hd)
- + ks_len(&merged_hdr->out_sq)
- + ks_len(&merged_hdr->out_rg)
- + ks_len(&merged_hdr->out_pg)
- + ks_len(&merged_hdr->out_co));
- if (txt_sz >= INT32_MAX) {
- fprintf(samtools_stderr, "[%s] Output header text too long\n", __func__);
- return NULL;
- }
-
- // Allocate new header
- hdr = bam_hdr_init();
- if (hdr == NULL) goto memfail;
-
- // Transfer targets arrays to new header
- hdr->n_targets = merged_hdr->n_targets;
- if (hdr->n_targets > 0) {
- // Try to shrink targets arrays to correct size
- hdr->target_name = realloc(merged_hdr->target_name,
- hdr->n_targets * sizeof(char*));
- if (!hdr->target_name) hdr->target_name = merged_hdr->target_name;
-
- hdr->target_len = realloc(merged_hdr->target_len,
- hdr->n_targets * sizeof(uint32_t));
- if (!hdr->target_len) hdr->target_len = merged_hdr->target_len;
-
- // These have either been freed by realloc() or, in the unlikely
- // event that failed, have had their ownership transferred to hdr
- merged_hdr->target_name = NULL;
- merged_hdr->target_len = NULL;
- }
- else {
- hdr->target_name = NULL;
- hdr->target_len = NULL;
- }
-
- // Allocate text
- text = hdr->text = malloc(txt_sz + 1);
- if (!text) goto memfail;
-
- // Put header text in order @HD, @SQ, @RG, @PG, @CO
- move_kstr_to_text(&text, &merged_hdr->out_hd);
- move_kstr_to_text(&text, &merged_hdr->out_sq);
- move_kstr_to_text(&text, &merged_hdr->out_rg);
- move_kstr_to_text(&text, &merged_hdr->out_pg);
- move_kstr_to_text(&text, &merged_hdr->out_co);
- hdr->l_text = txt_sz;
-
- return hdr;
+static int finish_merged_header(merged_header_t *merged_hdr) {
+ if (sam_hdr_add_lines(merged_hdr->hdr, ks_c_str(&merged_hdr->out_rg),
+ ks_len(&merged_hdr->out_rg)) < 0)
+ return -1;
+ if (sam_hdr_add_lines(merged_hdr->hdr, ks_c_str(&merged_hdr->out_pg),
+ ks_len(&merged_hdr->out_pg)) < 0)
+ return -1;
+ if (sam_hdr_add_lines(merged_hdr->hdr, ks_c_str(&merged_hdr->out_co),
+ ks_len(&merged_hdr->out_co)) < 0)
+ return -1;
- memfail:
- perror(__func__);
- bam_hdr_destroy(hdr);
- return NULL;
+ return 0;
}
/*
size_t i;
khiter_t iter;
if (!merged_hdr) return;
- free(ks_release(&merged_hdr->out_hd));
- free(ks_release(&merged_hdr->out_sq));
free(ks_release(&merged_hdr->out_rg));
free(ks_release(&merged_hdr->out_pg));
free(ks_release(&merged_hdr->out_co));
@param cmd command name (used in print_error() etc)
@param in_fmt format options for input files
@param out_fmt output file format and options
+ @param write_index create the index, together with the output file
+ @param arg_list command string for PG line
+ @param no_pg if 1, do not add a new PG line
@discussion Padding information may NOT correctly maintained. This
function is NOT thread safe.
*/
int bam_merge_core2(int by_qname, char* sort_tag, const char *out, const char *mode,
- const char *headers, int n, char * const *fn, int flag,
- const char *reg, int n_threads, const char *cmd,
- const htsFormat *in_fmt, const htsFormat *out_fmt)
+ const char *headers, int n, char * const *fn, char * const *fn_idx,
+ int flag, const char *reg, int n_threads, const char *cmd,
+ const htsFormat *in_fmt, const htsFormat *out_fmt, int write_index,
+ char *arg_list, int no_pg)
{
samFile *fpout, **fp = NULL;
heap1_t *heap = NULL;
- bam_hdr_t *hout = NULL;
- bam_hdr_t *hin = NULL;
+ sam_hdr_t *hout = NULL;
+ sam_hdr_t *hin = NULL;
int i, j, *RG_len = NULL;
uint64_t idx = 0;
char **RG = NULL;
hts_itr_t **iter = NULL;
- bam_hdr_t **hdr = NULL;
+ sam_hdr_t **hdr = NULL;
trans_tbl_t *translation_tbl = NULL;
int *rtrans = NULL;
+ char *out_idx_fn = NULL;
merged_header_t *merged_hdr = init_merged_header();
if (!merged_hdr) return -1;
if (sort_tag) {
g_is_by_tag = 1;
g_sort_tag[0] = sort_tag[0];
- g_sort_tag[1] = sort_tag[1];
+ g_sort_tag[1] = sort_tag[0] ? sort_tag[1] : '\0';
}
fp = (samFile**)calloc(n, sizeof(samFile*));
if (!heap) goto mem_fail;
iter = (hts_itr_t**)calloc(n, sizeof(hts_itr_t*));
if (!iter) goto mem_fail;
- hdr = (bam_hdr_t**)calloc(n, sizeof(bam_hdr_t*));
+ hdr = (sam_hdr_t**)calloc(n, sizeof(sam_hdr_t*));
if (!hdr) goto mem_fail;
translation_tbl = (trans_tbl_t*)calloc(n, sizeof(trans_tbl_t));
if (!translation_tbl) goto mem_fail;
// open and read the header from each file
for (i = 0; i < n; ++i) {
- bam_hdr_t *hin;
+ sam_hdr_t *hin;
fp[i] = sam_open_format(fn[i], "r", in_fmt);
if (fp[i] == NULL) {
print_error_errno(cmd, "fail to open \"%s\"", fn[i]);
// TODO sam_itr_next() doesn't yet work for SAM files,
// so for those keep the headers around for use with sam_read1()
if (hts_get_format(fp[i])->format == sam) hdr[i] = hin;
- else { bam_hdr_destroy(hin); hdr[i] = NULL; }
+ else { sam_hdr_destroy(hin); hdr[i] = NULL; }
if ((translation_tbl+i)->lost_coord_sort && !by_qname) {
fprintf(samtools_stderr, "[bam_merge_core] Order of targets in file %s caused coordinate sort to be lost\n", fn[i]);
}
// Transform the header into standard form
- hout = finish_merged_header(merged_hdr);
+ if (finish_merged_header(merged_hdr) < 0)
+ goto fail;
+
+ hout = merged_hdr->hdr;
if (!hout) return -1; // FIXME: memory leak
// If we're only merging a specified region move our iters to start at that point
if (reg) {
- int tid, beg, end;
- const char *name_lim;
+ int tid;
+ hts_pos_t beg, end;
- rtrans = rtrans_build(n, hout->n_targets, translation_tbl);
+ rtrans = rtrans_build(n, sam_hdr_nref(hout), translation_tbl);
if (!rtrans) goto mem_fail;
- name_lim = hts_parse_reg(reg, &beg, &end);
- if (name_lim) {
- char *name = malloc(name_lim - reg + 1);
- if (!name) goto mem_fail;
- memcpy(name, reg, name_lim - reg);
- name[name_lim - reg] = '\0';
- tid = bam_name2id(hout, name);
- free(name);
- }
- else {
- // not parsable as a region, but possibly a sequence named "foo:a"
- tid = bam_name2id(hout, reg);
- beg = 0;
- end = INT_MAX;
- }
- if (tid < 0) {
- if (name_lim) fprintf(samtools_stderr, "[%s] Region \"%s\" specifies an unknown reference name\n", __func__, reg);
- else fprintf(samtools_stderr, "[%s] Badly formatted region: \"%s\"\n", __func__, reg);
+ if (!sam_parse_region(hout, reg, &tid, &beg, &end, 0)) {
+ fprintf(samtools_stderr, "[%s] Badly formatted region or unknown reference name: \"%s\"\n", __func__, reg);
goto fail;
}
for (i = 0; i < n; ++i) {
- hts_idx_t *idx = sam_index_load(fp[i], fn[i]);
+ hts_idx_t *idx = NULL;
+ // If index filename has not been specfied, look in BAM folder
+ if (fn_idx != NULL) {
+ idx = sam_index_load2(fp[i], fn[i], fn_idx[i]);
+ } else {
+ idx = sam_index_load(fp[i], fn[i]);
+ }
// (rtrans[i*n+tid]) Look up what hout tid translates to in input tid space
- int mapped_tid = rtrans[i*hout->n_targets+tid];
+ int mapped_tid = rtrans[i*sam_hdr_nref(hout)+tid];
if (idx == NULL) {
fprintf(samtools_stderr, "[%s] failed to load index for %s. Random alignment retrieval only works for indexed BAM or CRAM files.\n",
__func__, fn[i]);
if (mapped_tid != INT32_MIN) {
fprintf(samtools_stderr,
"[%s] failed to get iterator over "
- "{%s, %d, %d, %d}\n",
+ "{%s, %d, %"PRIhts_pos", %"PRIhts_pos"}\n",
__func__, fn[i], mapped_tid, beg, end);
} else {
fprintf(samtools_stderr,
res = iter[i] ? sam_itr_next(fp[i], iter[i], h->entry.bam_record) : sam_read1(fp[i], hdr[i], h->entry.bam_record);
if (res >= 0) {
bam_translate(h->entry.bam_record, translation_tbl + i);
- h->pos = ((uint64_t)h->entry.bam_record->core.tid<<32) | (uint32_t)((int32_t)h->entry.bam_record->core.pos+1);
+ h->tid = h->entry.bam_record->core.tid;
+ h->pos = (uint64_t)(h->entry.bam_record->core.pos + 1);
h->rev = bam_is_rev(h->entry.bam_record);
h->idx = idx++;
if (g_is_by_tag) {
print_error_errno(cmd, "failed to create \"%s\"", out);
return -1;
}
+ if (!no_pg && sam_hdr_add_pg(hout, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error(cmd, "failed to add PG line to the header of \"%s\"", out);
+ sam_close(fpout);
+ return -1;
+ }
if (sam_hdr_write(fpout, hout) != 0) {
print_error_errno(cmd, "failed to write header to \"%s\"", out);
sam_close(fpout);
return -1;
}
+ if (write_index) {
+ if (!(out_idx_fn = auto_index(fpout, out, hout))){
+ sam_close(fpout);
+ return -1;
+ }
+ }
if (!(flag & MERGE_UNCOMP)) hts_set_threads(fpout, n_threads);
// Begin the actual merge
if (sam_write1(fpout, hout, b) < 0) {
print_error_errno(cmd, "failed writing to \"%s\"", out);
sam_close(fpout);
+ free(out_idx_fn);
return -1;
}
if ((j = (iter[heap->i]? sam_itr_next(fp[heap->i], iter[heap->i], b) : sam_read1(fp[heap->i], hdr[heap->i], b))) >= 0) {
bam_translate(b, translation_tbl + heap->i);
- heap->pos = ((uint64_t)b->core.tid<<32) | (uint32_t)((int)b->core.pos+1);
+ heap->tid = b->core.tid;
+ heap->pos = (uint64_t)(b->core.pos + 1);
heap->rev = bam_is_rev(b);
heap->idx = idx++;
if (g_is_by_tag) {
ks_heapadjust(heap, 0, n, heap);
}
+ if (write_index) {
+ if (sam_idx_save(fpout) < 0) {
+ print_error_errno("merge", "writing index failed");
+ goto fail;
+ }
+ }
+ free(out_idx_fn);
+
// Clean up and close
if (flag & MERGE_RG) {
for (i = 0; i != n; ++i) free(RG[i]);
for (i = 0; i < n; ++i) {
trans_tbl_destroy(translation_tbl + i);
hts_itr_destroy(iter[i]);
- bam_hdr_destroy(hdr[i]);
+ sam_hdr_destroy(hdr[i]);
sam_close(fp[i]);
}
- bam_hdr_destroy(hin);
- bam_hdr_destroy(hout);
+ sam_hdr_destroy(hin);
+ sam_hdr_destroy(hout);
free_merged_header(merged_hdr);
free(RG); free(translation_tbl); free(fp); free(heap); free(iter); free(hdr);
if (sam_close(fpout) < 0) {
for (i = 0; i < n; ++i) {
if (translation_tbl && translation_tbl[i].tid_trans) trans_tbl_destroy(translation_tbl + i);
if (iter && iter[i]) hts_itr_destroy(iter[i]);
- if (hdr && hdr[i]) bam_hdr_destroy(hdr[i]);
+ if (hdr && hdr[i]) sam_hdr_destroy(hdr[i]);
if (fp && fp[i]) sam_close(fp[i]);
if (heap && heap[i].entry.bam_record) bam_destroy1(heap[i].entry.bam_record);
}
- if (hout) bam_hdr_destroy(hout);
+ if (hout) sam_hdr_destroy(hout);
free(RG);
free(translation_tbl);
free(hdr);
free(heap);
free(fp);
free(rtrans);
+ free(out_idx_fn);
return -1;
}
strcpy(mode, "wb");
if (flag & MERGE_UNCOMP) strcat(mode, "0");
else if (flag & MERGE_LEVEL1) strcat(mode, "1");
- return bam_merge_core2(by_qname, NULL, out, mode, headers, n, fn, flag, reg, 0, "merge", NULL, NULL);
+ return bam_merge_core2(by_qname, NULL, out, mode, headers, n, fn, NULL, flag, reg, 0, "merge", NULL, NULL, 0, NULL, 1);
}
static void merge_usage(FILE *to)
" -c Combine @RG headers with colliding IDs [alter IDs to be distinct]\n"
" -p Combine @PG headers with colliding IDs [alter IDs to be distinct]\n"
" -s VALUE Override random seed\n"
-" -b FILE List of input BAM filenames, one per line [null]\n");
- sam_global_opt_help(to, "-.O..@");
+" -b FILE List of input BAM filenames, one per line [null]\n"
+" -X Use customized index files\n"
+" --no-PG do not add a PG line\n");
+ sam_global_opt_help(to, "-.O..@..");
}
int bam_merge(int argc, char *argv[])
{
- int c, is_by_qname = 0, flag = 0, ret = 0, level = -1;
+ int c, is_by_qname = 0, flag = 0, ret = 0, level = -1, has_index_file = 0;
char *fn_headers = NULL, *reg = NULL, mode[12];
- char *sort_tag = NULL;
+ char *sort_tag = NULL, *arg_list = NULL;
long random_seed = (long)time(NULL);
char** fn = NULL;
- int fn_size = 0;
+ char** fn_idx = NULL;
+ int fn_size = 0, no_pg = 0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 0, '@'),
{ "threads", required_argument, NULL, '@' },
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
return 0;
}
- while ((c = getopt_long(argc, argv, "h:nru1R:f@:l:cps:b:O:t:", lopts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "h:nru1R:f@:l:cps:b:O:t:X", lopts, NULL)) >= 0) {
switch (c) {
case 'r': flag |= MERGE_RG; break;
case 'f': flag |= MERGE_FORCE; break;
- case 'h': fn_headers = strdup(optarg); break;
+ case 'h': fn_headers = optarg; break;
case 'n': is_by_qname = 1; break;
- case 't': sort_tag = strdup(optarg); break;
+ case 't': sort_tag = optarg; break;
case '1': flag |= MERGE_LEVEL1; level = 1; break;
case 'u': flag |= MERGE_UNCOMP; level = 0; break;
case 'R': reg = strdup(optarg); break;
case 'c': flag |= MERGE_COMBINE_RG; break;
case 'p': flag |= MERGE_COMBINE_PG; break;
case 's': random_seed = atol(optarg); break;
+ case 'X': has_index_file = 1; break; // -X flag for index filename
case 'b': {
// load the list of files to read
+ if (has_index_file) {
+ fprintf(samtools_stderr,"Error: The -b option cannot be combined with -X\n");
+ ret = 1; goto end;
+ }
int nfiles;
char **fn_read = hts_readlines(optarg, &nfiles);
if (fn_read) {
}
break;
}
-
+ case 1: no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': merge_usage(samtools_stderr); return 1;
return 1;
}
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("merge", "failed to create arg_list");
+ return 1;
+ }
+
srand48(random_seed);
if (!(flag & MERGE_FORCE) && strcmp(argv[optind], "-")) {
FILE *fp = fopen(argv[optind], "rb");
}
}
- int nargcfiles = argc - (optind+1);
+ int nargcfiles = 0;
+ if (has_index_file) { // Calculate # of input BAM files
+ if ((argc - optind - 1) % 2 != 0) {
+ fprintf(samtools_stderr, "Odd number of filenames detected! Each BAM file should have an index file\n");
+ return 1;
+ }
+ nargcfiles = (argc - optind - 1) / 2;
+ } else {
+ nargcfiles = argc - optind - 1;
+ }
+
if (nargcfiles > 0) {
// Add argc files to end of array
fn = realloc(fn, (fn_size+nargcfiles) * sizeof(char*));
if (fn == NULL) { ret = 1; goto end; }
memcpy(fn+fn_size, argv + (optind+1), nargcfiles * sizeof(char*));
+
+ if(has_index_file) {
+ fn_idx = realloc(fn_idx, nargcfiles * sizeof(char*));
+ if (fn_idx == NULL) { ret = 1; goto end; }
+ memcpy(fn_idx+fn_size, argv + nargcfiles + (optind+1), nargcfiles * sizeof(char*));
+ }
}
if (fn_size+nargcfiles < 1) {
print_error("merge", "You must specify at least one (and usually two or more) input files");
merge_usage(samtools_stderr);
+ free(fn_idx);
return 1;
}
strcpy(mode, "wb");
sam_open_mode(mode+1, argv[optind], NULL);
if (level >= 0) sprintf(strchr(mode, '\0'), "%d", level < 9? level : 9);
if (bam_merge_core2(is_by_qname, sort_tag, argv[optind], mode, fn_headers,
- fn_size+nargcfiles, fn, flag, reg, ga.nthreads,
- "merge", &ga.in, &ga.out) < 0)
+ fn_size+nargcfiles, fn, fn_idx, flag, reg, ga.nthreads,
+ "merge", &ga.in, &ga.out, ga.write_index, arg_list, no_pg) < 0)
ret = 1;
end:
for (i=0; i<fn_size; i++) free(fn[i]);
}
free(fn);
+ free(fn_idx);
free(reg);
- free(fn_headers);
+ free(arg_list);
sam_global_args_free(&ga);
return ret;
}
static inline int heap_add_read(heap1_t *heap, int nfiles, samFile **fp,
int num_in_mem, buf_region *in_mem,
- bam1_tag *buf, uint64_t *idx, bam_hdr_t *hout) {
+ bam1_tag *buf, uint64_t *idx, sam_hdr_t *hout) {
int i = heap->i, res;
if (i < nfiles) { // read from file
res = sam_read1(fp[i], hout, heap->entry.bam_record);
}
}
if (res >= 0) {
- heap->pos = (((uint64_t)heap->entry.bam_record->core.tid<<32)
- | (uint32_t)((int32_t)heap->entry.bam_record->core.pos+1));
+ heap->tid = heap->entry.bam_record->core.tid;
+ heap->pos = (uint64_t)(heap->entry.bam_record->core.pos + 1);
heap->rev = bam_is_rev(heap->entry.bam_record);
heap->idx = (*idx)++;
if (g_is_by_tag) {
}
static int bam_merge_simple(int by_qname, char *sort_tag, const char *out,
- const char *mode, bam_hdr_t *hout,
+ const char *mode, sam_hdr_t *hout,
int n, char * const *fn, int num_in_mem,
buf_region *in_mem, bam1_tag *buf, int n_threads,
const char *cmd, const htsFormat *in_fmt,
- const htsFormat *out_fmt) {
+ const htsFormat *out_fmt, char *arg_list, int no_pg,
+ int write_index) {
samFile *fpout = NULL, **fp = NULL;
heap1_t *heap = NULL;
uint64_t idx = 0;
int i, heap_size = n + num_in_mem;
+ char *out_idx_fn = NULL;
g_is_by_qname = by_qname;
if (sort_tag) {
g_is_by_tag = 1;
g_sort_tag[0] = sort_tag[0];
- g_sort_tag[1] = sort_tag[1];
+ g_sort_tag[1] = sort_tag[0] ? sort_tag[1] : '\0';
}
if (n > 0) {
fp = (samFile**)calloc(n, sizeof(samFile*));
// Open each file, read the header and put the first read into the heap
for (i = 0; i < heap_size; i++) {
- bam_hdr_t *hin;
+ sam_hdr_t *hin;
heap1_t *h = &heap[i];
if (i < n) {
goto fail;
}
// ... and throw it away as we don't really need it
- bam_hdr_destroy(hin);
+ sam_hdr_destroy(hin);
}
// Get a read into the heap
return -1;
}
+ if (!no_pg && sam_hdr_add_pg(hout, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error(cmd, "failed to add PG line to the header of \"%s\"", out);
+ sam_close(fpout);
+ return -1;
+ }
+
if (n_threads > 1) hts_set_threads(fpout, n_threads);
if (sam_hdr_write(fpout, hout) != 0) {
return -1;
}
+ if (write_index) {
+ if (!(out_idx_fn = auto_index(fpout, out, hout))){
+ sam_close(fpout);
+ return -1;
+ }
+ }
+
// Now do the merge
ks_heapmake(heap, heap_size, heap);
while (heap->pos != HEAP_EMPTY) {
bam1_t *b = heap->entry.bam_record;
if (sam_write1(fpout, hout, b) < 0) {
print_error_errno(cmd, "failed writing to \"%s\"", out);
- sam_close(fpout);
- return -1;
+ goto fail;
}
if (heap_add_read(heap, n, fp, num_in_mem, in_mem, buf, &idx, hout) < 0) {
assert(heap->i < n);
}
free(fp);
free(heap);
+
+ if (write_index) {
+ if (sam_idx_save(fpout) < 0) {
+ print_error_errno("merge", "writing index failed");
+ goto fail;
+ }
+ free(out_idx_fn);
+ }
+
if (sam_close(fpout) < 0) {
print_error(cmd, "error closing output file");
return -1;
fail:
for (i = 0; i < n; i++) {
if (fp && fp[i]) sam_close(fp[i]);
- if (heap && heap[i].entry.bam_record) bam_destroy1(heap[i].entry.bam_record);
+ }
+ for (i = 0; i < heap_size; i++) {
+ if (heap && heap[i].i < n && heap[i].entry.bam_record)
+ bam_destroy1(heap[i].entry.bam_record);
}
free(fp);
free(heap);
if (fpout) sam_close(fpout);
+ free(out_idx_fn);
return -1;
}
if (t != 0) return t;
return (int) (a.bam_record->core.flag&0xc0) - (int) (b.bam_record->core.flag&0xc0);
} else {
- pa = (uint64_t)a.bam_record->core.tid<<32|(a.bam_record->core.pos+1);
- pb = (uint64_t)b.bam_record->core.tid<<32|(b.bam_record->core.pos+1);
+ pa = a.bam_record->core.tid;
+ pb = b.bam_record->core.tid;
+
+ if (pa == pb) {
+ pa = (uint64_t)(a.bam_record->core.pos+1);
+ pb = (uint64_t)(b.bam_record->core.pos+1);
+ }
if (pa == pb) {
pa = bam_is_rev(a.bam_record);
size_t buf_len;
const char *prefix;
bam1_tag *buf;
- const bam_hdr_t *h;
+ const sam_hdr_t *h;
int index;
int error;
int no_save;
// Returns 0 for success
// -1 for failure
-static int write_buffer(const char *fn, const char *mode, size_t l, bam1_tag *buf, const bam_hdr_t *h, int n_threads, const htsFormat *fmt)
+static int write_buffer(const char *fn, const char *mode, size_t l, bam1_tag *buf,
+ const sam_hdr_t *h, int n_threads, const htsFormat *fmt,
+ char *arg_list, int no_pg, int write_index)
{
size_t i;
samFile* fp;
+ char *out_idx_fn = NULL;
+
fp = sam_open_format(fn, mode, fmt);
if (fp == NULL) return -1;
- if (sam_hdr_write(fp, h) != 0) goto fail;
+ if (!no_pg && sam_hdr_add_pg((sam_hdr_t *)h, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ goto fail;
+ }
+ if (sam_hdr_write(fp, (sam_hdr_t *)h) != 0) goto fail;
+
+ if (write_index) {
+ if (!(out_idx_fn = auto_index(fp, fn, (sam_hdr_t *)h))) goto fail;
+ }
+
if (n_threads > 1) hts_set_threads(fp, n_threads);
for (i = 0; i < l; ++i) {
- if (sam_write1(fp, h, buf[i].bam_record) < 0) goto fail;
+ if (sam_write1(fp, (sam_hdr_t *)h, buf[i].bam_record) < 0) goto fail;
}
+
+ if (write_index) {
+ if (sam_idx_save(fp) < 0) {
+ print_error_errno("merge", "writing index failed");
+ goto fail;
+ }
+ free(out_idx_fn);
+ }
+
+
if (sam_close(fp) < 0) return -1;
return 0;
fail:
sam_close(fp);
+ free(out_idx_fn);
return -1;
}
#define NUMBASE 256
-#define STEP 8
-static int ks_radixsort(size_t n, bam1_tag *buf, const bam_hdr_t *h)
+static int ks_radixsort(size_t n, bam1_tag *buf, const sam_hdr_t *h)
{
int curr = 0, ret = -1;
ssize_t i;
bam1_tag *buf_ar2[2], *bam_a, *bam_b;
- uint64_t max_pos = 0, max_digit = 0, shift = 0;
-
+ uint64_t max_pos = 1;
+ uint32_t max_tid = 1, tid_bytes = 0, pos_bytes = 0, byte = 0;
+ uint32_t tid_shift_l, tid_shift_r;
+ int nref = sam_hdr_nref(h);
+
+ // Count number of bytes needed for biggest tid and pos
+ // Notes: Add 1 to core.pos so always positive.
+ // Convert unmapped tid (-1) to number of references so unmapped
+ // sort to the end.
for (i = 0; i < n; i++) {
bam1_t *b = buf[i].bam_record;
- int32_t tid = b->core.tid == -1 ? h->n_targets : b->core.tid;
- buf[i].u.pos = (uint64_t)tid<<32 | (b->core.pos+1)<<1 | bam_is_rev(b);
- if (max_pos < buf[i].u.pos)
- max_pos = buf[i].u.pos;
+ uint32_t tid = b->core.tid == -1 ? nref : b->core.tid;
+ uint64_t pos = ((uint64_t)(b->core.pos + 1) << 1) | bam_is_rev(b);
+ if (max_tid < tid)
+ max_tid = tid;
+ if (max_pos < pos)
+ max_pos = pos;
}
- while (max_pos) {
- ++max_digit;
- max_pos = max_pos >> 1;
+ for (; max_pos > 0; max_pos >>= 8) pos_bytes++;
+ for (; max_tid > 0; max_tid >>= 8) tid_bytes++;
+ assert(pos_bytes + tid_bytes < sizeof(buf[0].u.pos_tid));
+
+ tid_shift_l = pos_bytes * 8;
+ tid_shift_r = 64 - tid_shift_l;
+
+ // Write position and tid into bam1_tag::u::pos_tid using minimum number
+ // of bytes required. Values are stored little-endian so that we
+ // get a least-significant digit (byte) radix sort.
+ for (i = 0; i < n; i++) {
+ bam1_t *b = buf[i].bam_record;
+ uint32_t tid = b->core.tid == -1 ? nref : b->core.tid;
+ // 'pos' here includes as many bytes of tid as will fit
+ // in the space remaining above pos_bytes. The rest of tid
+ // is written out separately.
+ uint64_t pos = (bam_is_rev(b) |
+ ((uint64_t)(b->core.pos + 1) << 1) |
+ (tid_shift_l < 64 ? (uint64_t) tid << tid_shift_l : 0));
+ u64_to_le(pos, buf[i].u.pos_tid);
+ u32_to_le(tid_shift_r < 32 ? tid >> tid_shift_r : 0,
+ &buf[i].u.pos_tid[8]);
}
buf_ar2[0] = buf;
goto err;
}
- while (shift < max_digit){
+ // Least-significant digit radix sort (where "digits" are bytes)
+ for (byte = 0; byte < pos_bytes + tid_bytes; byte++) {
size_t remainders[NUMBASE] = { 0 };
bam_a = buf_ar2[curr]; bam_b = buf_ar2[1-curr];
for (i = 0; i < n; ++i)
- remainders[(bam_a[i].u.pos >> shift) % NUMBASE]++;
+ remainders[bam_a[i].u.pos_tid[byte]]++;
for (i = 1; i < NUMBASE; ++i)
remainders[i] += remainders[i - 1];
for (i = n - 1; i >= 0; i--) {
- size_t j = --remainders[(bam_a[i].u.pos >> shift) % NUMBASE];
+ size_t j = --remainders[bam_a[i].u.pos_tid[byte]];
bam_b[j] = bam_a[i];
}
- shift += STEP;
curr = 1 - curr;
}
if (curr == 1) {
return 0;
}
- if (write_buffer(name, "wcx1", w->buf_len, w->buf, w->h, 0, &fmt) < 0)
+ if (write_buffer(name, "wcx1", w->buf_len, w->buf, w->h, 0, &fmt, NULL, 1, 0) < 0)
w->error = errno;
} else {
- if (write_buffer(name, "wbx1", w->buf_len, w->buf, w->h, 0, NULL) < 0)
+ if (write_buffer(name, "wbx1", w->buf_len, w->buf, w->h, 0, NULL, NULL, 1, 0) < 0)
w->error = errno;
}
}
static int sort_blocks(int n_files, size_t k, bam1_tag *buf, const char *prefix,
- const bam_hdr_t *h, int n_threads, buf_region *in_mem)
+ const sam_hdr_t *h, int n_threads, buf_region *in_mem)
{
int i;
size_t pos, rest;
@param max_mem approxiate maximum memory (very inaccurate)
@param in_fmt input file format options
@param out_fmt output file format and options
+ @param arg_list command string for PG line
+ @param no_pg if 1, do not add a new PG line
+ @paran write_index create index for the output file
@return 0 for successful sorting, negative on errors
@discussion It may create multiple temporary subalignment files
int bam_sort_core_ext(int is_by_qname, char* sort_by_tag, const char *fn, const char *prefix,
const char *fnout, const char *modeout,
size_t _max_mem, int n_threads,
- const htsFormat *in_fmt, const htsFormat *out_fmt)
+ const htsFormat *in_fmt, const htsFormat *out_fmt,
+ char *arg_list, int no_pg, int write_index)
{
int ret = -1, res, i, n_files = 0;
size_t max_k, k, max_mem, bam_mem_offset;
- bam_hdr_t *header = NULL;
+ sam_hdr_t *header = NULL;
samFile *fp;
bam1_tag *buf = NULL;
bam1_t *b = bam_init1();
g_is_by_qname = is_by_qname;
if (sort_by_tag) {
g_is_by_tag = 1;
- strncpy(g_sort_tag, sort_by_tag, 2);
+ g_sort_tag[0] = sort_by_tag[0];
+ g_sort_tag[1] = sort_by_tag[0] ? sort_by_tag[1] : '\0';
}
max_mem = _max_mem * n_threads;
else
new_so = "coordinate";
- if (sam_hdr_change_HD(header, "SO", new_so) != 0) {
- print_error("sort",
- "failed to change sort order header to '%s'\n", new_so);
+ if ((-1 == sam_hdr_update_hd(header, "SO", new_so))
+ && (-1 == sam_hdr_add_line(header, "HD", "VN", SAM_FORMAT_VERSION, "SO", new_so, NULL))
+ ) {
+ print_error("sort", "failed to change sort order header to '%s'\n", new_so);
goto err;
}
- if (sam_hdr_change_HD(header, "GO", NULL) != 0) {
- print_error("sort",
- "failed to delete group order header\n");
+
+ if (-1 == sam_hdr_remove_tag_hd(header, "GO")) {
+ print_error("sort", "failed to delete group order header\n");
goto err;
}
// write the final output
if (n_files == 0 && num_in_mem < 2) { // a single block
- if (write_buffer(fnout, modeout, k, buf, header, n_threads, out_fmt) != 0) {
+ if (write_buffer(fnout, modeout, k, buf, header, n_threads, out_fmt, arg_list, no_pg, write_index) != 0) {
print_error_errno("sort", "failed to create \"%s\"", fnout);
goto err;
}
}
if (bam_merge_simple(is_by_qname, sort_by_tag, fnout, modeout, header,
n_files, fns, num_in_mem, in_mem, buf,
- n_threads, "sort", in_fmt, out_fmt) < 0) {
+ n_threads, "sort", in_fmt, out_fmt, arg_list,
+ no_pg, write_index) < 0) {
// Propagate bam_merge_simple() failure; it has already emitted a
// message explaining the failure, so no further message is needed.
goto err;
free(buf);
free(bam_mem);
free(in_mem);
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
if (fp) sam_close(fp);
return ret;
}
char *fnout = calloc(strlen(prefix) + 4 + 1, 1);
if (!fnout) return -1;
sprintf(fnout, "%s.bam", prefix);
- ret = bam_sort_core_ext(is_by_qname, NULL, fn, prefix, fnout, "wb", max_mem, 0, NULL, NULL);
+ ret = bam_sort_core_ext(is_by_qname, NULL, fn, prefix, fnout, "wb", max_mem, 0, NULL, NULL, NULL, 1, 0);
free(fnout);
return ret;
}
" -n Sort by read name\n"
" -t TAG Sort by value of TAG. Uses position as secondary index (or read name if -n is set)\n"
" -o FILE Write final output to FILE rather than standard output\n"
-" -T PREFIX Write temporary files to PREFIX.nnnn.bam\n");
- sam_global_opt_help(fp, "-.O..@");
+" -T PREFIX Write temporary files to PREFIX.nnnn.bam\n"
+" --no-PG do not add a PG line\n");
+ sam_global_opt_help(fp, "-.O..@-.");
}
static void complain_about_memory_setting(size_t max_mem) {
int bam_sort(int argc, char *argv[])
{
size_t max_mem = SORT_DEFAULT_MEGS_PER_THREAD << 20;
- int c, nargs, is_by_qname = 0, ret, o_seen = 0, level = -1;
- char* sort_tag = NULL;
+ int c, nargs, is_by_qname = 0, ret, o_seen = 0, level = -1, no_pg = 0;
+ char* sort_tag = NULL, *arg_list = NULL;
char *fnout = "-", modeout[12];
kstring_t tmpprefix = { 0, 0, NULL };
struct stat st;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 0, '@'),
{ "threads", required_argument, NULL, '@' },
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
switch (c) {
case 'o': fnout = optarg; o_seen = 1; break;
case 'n': is_by_qname = 1; break;
- case 't': sort_tag = strdup(optarg); break;
+ case 't': sort_tag = optarg; break;
case 'm': {
char *q;
max_mem = strtol(optarg, &q, 0);
}
case 'T': kputs(optarg, &tmpprefix); break;
case 'l': level = atoi(optarg); break;
+ case 1: no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
goto sort_end;
}
+ if (ga.write_index && (is_by_qname || sort_tag)) {
+ fprintf(samtools_stderr, "[W::bam_sort] Ignoring --write-index as it only works for position sorted files.\n");
+ ga.write_index = 0;
+ }
+
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("sort", "failed to create arg_list");
+ return 1;
+ }
+
if (max_mem < (SORT_MIN_MEGS_PER_THREAD << 20)) {
complain_about_memory_setting(max_mem);
ret = EXIT_FAILURE;
ret = bam_sort_core_ext(is_by_qname, sort_tag, (nargs > 0)? argv[optind] : "-",
tmpprefix.s, fnout, modeout, max_mem, ga.nthreads,
- &ga.in, &ga.out);
+ &ga.in, &ga.out, arg_list, no_pg, ga.write_index);
if (ret >= 0)
ret = EXIT_SUCCESS;
else {
sort_end:
free(tmpprefix.s);
+ free(arg_list);
sam_global_args_free(&ga);
return ret;
/* bam_split.c -- split subcommand.
- Copyright (C) 2013-2016 Genome Research Ltd.
+ Copyright (C) 2013-2016,2018-2019 Genome Research Ltd.
Author: Martin Pollard <mp15@sanger.ac.uk>
#include <config.h>
-#include <htslib/sam.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <limits.h>
#include <unistd.h>
#include <regex.h>
+#include <assert.h>
+#include <htslib/sam.h>
#include <htslib/khash.h>
#include <htslib/kstring.h>
#include <htslib/cram.h>
KHASH_MAP_INIT_STR(c2i, int)
struct parsed_opts {
- char* merged_input_name;
- char* unaccounted_header_name;
- char* unaccounted_name;
- char* output_format_string;
+ const char *merged_input_name;
+ const char *unaccounted_header_name;
+ const char *unaccounted_name;
+ const char *output_format_string;
bool verbose;
+ int no_pg;
sam_global_args ga;
};
struct state {
samFile* merged_input_file;
- bam_hdr_t* merged_input_header;
+ sam_hdr_t* merged_input_header;
samFile* unaccounted_file;
- bam_hdr_t* unaccounted_header;
+ sam_hdr_t* unaccounted_header;
size_t output_count;
char** rg_id;
+ char **rg_index_file_name;
char **rg_output_file_name;
samFile** rg_output_file;
- bam_hdr_t** rg_output_header;
+ sam_hdr_t** rg_output_header;
kh_c2i_t* rg_hash;
htsThreadPool p;
+ int write_index;
};
typedef struct state state_t;
static void usage(FILE *write_to)
{
fprintf(write_to,
-"Usage: samtools split [-u <unaccounted.bam>[:<unaccounted_header.sam>]]\n"
+"Usage: samtools split [-u <unaccounted.bam>] [-h <unaccounted_header.sam>]\n"
" [-f <format_string>] [-v] <merged.bam>\n"
"Options:\n"
" -f STRING output filename format string [\"%%*_%%#.%%.\"]\n"
" -u FILE1 put reads with no RG tag or an unrecognised RG tag in FILE1\n"
-" -u FILE1:FILE2 ...and override the header with FILE2\n"
-" -v verbose output\n");
- sam_global_opt_help(write_to, "-....@");
+" -h FILE2 ... and override the header with FILE2 (-u file only)\n"
+" -v verbose output\n"
+" --no-PG do not add a PG line\n");
+ sam_global_opt_help(write_to, "-....@..");
fprintf(write_to,
"\n"
"Format string expansions:\n"
{
if (argc == 1) { usage(stdout); return NULL; }
- const char* optstring = "vf:u:@:";
- char* delim;
+ const char *optstring = "vf:h:u:@:";
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 0, '@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
while ((opt = getopt_long(argc, argv, optstring, lopts, NULL)) != -1) {
switch (opt) {
case 'f':
- retval->output_format_string = strdup(optarg);
- if (! retval->output_format_string ) { perror("cannot allocate output format string memory"); return NULL; }
+ retval->output_format_string = optarg;
+ break;
+ case 'h':
+ retval->unaccounted_header_name = optarg;
break;
case 'v':
retval->verbose = true;
break;
case 'u':
- retval->unaccounted_name = strdup(optarg);
- if (! retval->unaccounted_name ) { perror("cannot allocate string memory"); return NULL; }
- if ((delim = strchr(retval->unaccounted_name, ':')) != NULL) {
- *delim = '\0';
- retval->unaccounted_header_name = strdup(delim+1);
- if (! retval->unaccounted_header_name ) { perror("cannot allocate string memory"); return NULL; }
- }
+ retval->unaccounted_name = optarg;
+ break;
+ case 1:
+ retval->no_pg = 1;
break;
default:
if (parse_sam_global_opt(opt, optarg, lopts, &retval->ga) == 0) break;
}
}
- if (retval->output_format_string == NULL) retval->output_format_string = strdup("%*_%#.%.");
+ if (retval->output_format_string == NULL) retval->output_format_string = "%*_%#.%.";
argc -= optind;
argv += optind;
return NULL;
}
- retval->merged_input_name = strdup(argv[0]);
- if (! retval->merged_input_name ) { perror("cannot allocate string memory"); return NULL; }
+ retval->merged_input_name = argv[0];
return retval;
}
const char* pointer = format_string;
const char* next;
while ((next = strchr(pointer, '%')) != NULL) {
- kputsn(pointer, next-pointer, &str);
+ if (kputsn(pointer, next-pointer, &str) < 0) goto memfail;
++next;
switch (*next) {
case '%':
- kputc('%', &str);
+ if (kputc('%', &str) < 0) goto memfail;
break;
case '*':
- kputs(basename, &str);
+ if (kputs(basename, &str) < 0) goto memfail;
break;
case '#':
- kputl(rg_idx, &str);
+ if (kputl(rg_idx, &str) < 0) goto memfail;
break;
case '!':
- kputs(rg_id, &str);
+ if (kputs(rg_id, &str) < 0) goto memfail;
break;
case '.':
// Only really need to cope with sam, bam, cram
- if (format->format != unknown_format)
- kputs(hts_format_file_extension(format), &str);
- else
- kputs("bam", &str);
+ if (format->format != unknown_format) {
+ if (kputs(hts_format_file_extension(format), &str) < 0)
+ goto memfail;
+ } else {
+ if (kputs("bam", &str) < 0) goto memfail;
+ }
break;
case '\0':
- // Error is: fprintf(stderr, "bad format string, trailing %%\n");
- free(str.s);
- return NULL;
+ print_error("split", "Trailing %% in filename format string");
+ goto fail;
default:
// Error is: fprintf(stderr, "bad format string, unknown format specifier\n");
- free(str.s);
- return NULL;
+ print_error("split", "Unknown specifier %%%c in filename format string", *next);
+ goto fail;
}
pointer = next + 1;
}
- kputs(pointer, &str);
+ if (kputs(pointer, &str) < 0) goto memfail;
return ks_release(&str);
+
+ memfail:
+ print_error_errno("split", "Couldn't build output filename");
+ fail:
+ free(str.s);
+ return NULL;
}
// Parse the header, count the number of RG tags and return a list of their names
-static bool count_RG(bam_hdr_t* hdr, size_t* count, char*** output_name)
+static bool count_RG(sam_hdr_t* hdr, size_t* count, char*** output_name)
{
- if (hdr->l_text < 3 ) {
+ char **names = NULL;
+ kstring_t id_val = KS_INITIALIZE;
+ int i, n_rg = sam_hdr_count_lines(hdr, "RG");
+
+ if (n_rg < 0) {
+ print_error("split", "Failed to get @RG IDs");
*count = 0;
*output_name = NULL;
- return true;
- }
- kstring_t input = { 0, 0, NULL };
- kputsn(hdr->text, hdr->l_text, &input);
-
- //////////////////////////////////////////
- // First stage count number of @RG tags //
- //////////////////////////////////////////
- char* pointer = ks_str(&input);
- size_t n_rg = 0;
- // Guard against rare case where @RG is first header line
- // This shouldn't happen but could where @HD is omitted
- if (pointer[0] == '@' && pointer[1] == 'R' && pointer[2] == 'G' ) {
- ++n_rg;
- pointer += 3;
- }
- char* line;
- while ((line = strstr(pointer, "\n@RG")) != NULL) {
- ++n_rg;
- pointer = line + 1;
- }
-
- //////////////////////////////////
- // Second stage locate @RG ID's //
- //////////////////////////////////
- char** names = (char**)calloc(sizeof(char*), n_rg);
- size_t next = 0;
-
- regex_t rg_finder;
- if (regcomp(&rg_finder, "^@RG.*\tID:([!-)+-<>-~][ !-~]*)(\t.*$|$)", REG_EXTENDED|REG_NEWLINE) != 0) {
- free(input.s);
- free(names);
return false;
}
- regmatch_t* matches = (regmatch_t*)calloc(sizeof(regmatch_t),2);
- int error;
- char* begin = ks_str(&input);
-
- while ((error = regexec(&rg_finder, begin, 2, matches, 0)) == 0) {
- kstring_t str = { 0, 0, NULL };
- kputsn(begin+matches[1].rm_so, matches[1].rm_eo-matches[1].rm_so, &str);
- names[next++] = ks_release(&str);
- begin += matches[0].rm_eo;
+
+ if (n_rg == 0) {
+ *count = 0;
+ *output_name = NULL;
+ return true;
}
- if (error != REG_NOMATCH) {
- // cleanup
- regfree(&rg_finder);
- free(matches);
- free(names);
- free(input.s);
- return false;
+ names = calloc(n_rg, sizeof(names[0]));
+ if (!names) goto memfail;
+
+ for (i = 0; i < n_rg; i++) {
+ if (sam_hdr_find_tag_pos(hdr, "RG", i, "ID", &id_val) < 0) goto memfail;
+ names[i] = ks_release(&id_val);
}
- free(matches);
- // return results
*count = n_rg;
*output_name = names;
- regfree(&rg_finder);
- free(input.s);
return true;
+
+ memfail:
+ print_error_errno("split", "Failed to get @RG IDs");
+ *count = 0;
+ *output_name = NULL;
+ ks_free(&id_val);
+ free(names);
+ return false;
}
-// Filters a header of @RG lines where ID != id_keep
-// TODO: strip @PG's descended from other RGs and their descendants
-static bool filter_header_rg(bam_hdr_t* hdr, const char* id_keep, const char *arg_list)
+static int header_compatible(sam_hdr_t *hdr1, sam_hdr_t *hdr2)
{
- kstring_t str = {0, 0, NULL};
-
- regex_t rg_finder;
-
- if (regcomp(&rg_finder, "^@RG.*\tID:([!-)+-<>-~][ !-~]*)(\t.*$|$)", REG_EXTENDED|REG_NEWLINE) != 0) {
- return false;
+ size_t n;
+ if (sam_hdr_nref(hdr1) != sam_hdr_nref(hdr2)) {
+ print_error("split",
+ "Unaccounted header contains wrong number of references");
+ return -1;
}
-
- // regex vars
- char* header = hdr->text;
- regmatch_t* matches = (regmatch_t*)calloc(sizeof(regmatch_t),2);
- kstring_t found_id = { 0, 0, NULL };
- int error;
-
- while ((error = regexec(&rg_finder, header, 2, matches, 0)) == 0) {
- kputsn(header, matches[0].rm_so, &str); // copy header up until the found RG line
-
- found_id.l = 0;
- kputsn(header+matches[1].rm_so, matches[1].rm_eo-matches[1].rm_so, &found_id); // extract ID
- // if it matches keep keep it, else we can just ignore it
- if (strcmp(ks_str(&found_id), id_keep) == 0) {
- kputsn(header+matches[0].rm_so, (matches[0].rm_eo+1)-matches[0].rm_so, &str);
+ for (n = 0; n < sam_hdr_nref(hdr1); n++) {
+ hts_pos_t h1_len = sam_hdr_tid2len(hdr1, n);
+ hts_pos_t h2_len = sam_hdr_tid2len(hdr2, n);
+ if (h1_len != h2_len) {
+ print_error("split",
+ "Unaccounted header reference %zu \"%s\" is not the same length as in the input file",
+ n + 1, sam_hdr_tid2name(hdr2, n));
+ return -1;
}
- // move pointer forward
- header += matches[0].rm_eo+1;
- }
- // cleanup
- free(found_id.s);
- free(matches);
- regfree(&rg_finder);
- // Did we leave loop because of an error?
- if (error != REG_NOMATCH) {
- return false;
}
-
- // Write remainder of string
- kputs(header, &str);
-
- // Modify header
- hdr->l_text = ks_len(&str);
- free(hdr->text);
- hdr->text = ks_release(&str);
-
- // Add the PG line
- SAM_hdr *sh = sam_hdr_parse_(hdr->text, hdr->l_text);
- if (sam_hdr_add_PG(sh, "samtools",
- "VN", samtools_version(),
- arg_list ? "CL": NULL,
- arg_list ? arg_list : NULL,
- NULL) != 0)
- return -1;
-
- free(hdr->text);
- hdr->text = strdup(sam_hdr_str(sh));
- hdr->l_text = sam_hdr_length(sh);
- if (!hdr->text)
- return false;
- sam_hdr_free(sh);
-
- return true;
+ return 0;
}
// Set the initial state
if (opts->ga.nthreads > 0) {
if (!(retval->p.pool = hts_tpool_init(opts->ga.nthreads))) {
fprintf(stderr, "Error creating thread pool\n");
+ cleanup_state(retval, false);
return NULL;
}
}
retval->merged_input_file = sam_open_format(opts->merged_input_name, "rb", &opts->ga.in);
if (!retval->merged_input_file) {
print_error_errno("split", "Could not open \"%s\"", opts->merged_input_name);
- free(retval);
+ cleanup_state(retval, false);
return NULL;
}
if (retval->p.pool)
if (retval->unaccounted_header == NULL) {
print_error("split", "Could not read header from \"%s\"", opts->unaccounted_header_name);
cleanup_state(retval, false);
+ sam_close(hdr_load);
return NULL;
}
sam_close(hdr_load);
+ if (header_compatible(retval->merged_input_header,
+ retval->unaccounted_header) != 0) {
+ cleanup_state(retval, false);
+ return NULL;
+ }
} else {
- retval->unaccounted_header = bam_hdr_dup(retval->merged_input_header);
+ retval->unaccounted_header = sam_hdr_dup(retval->merged_input_header);
+ if (!opts->no_pg && sam_hdr_add_pg(retval->unaccounted_header, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error("split", "Could not rewrite header for \"%s\"", opts->unaccounted_name);
+ cleanup_state(retval, false);
+ return NULL;
+ }
}
retval->unaccounted_file = sam_open_format(opts->unaccounted_name, "wb", &opts->ga.out);
// Open output files for RGs
if (!count_RG(retval->merged_input_header, &retval->output_count, &retval->rg_id)) return NULL;
if (opts->verbose) fprintf(stderr, "@RG's found %zu\n",retval->output_count);
-
- retval->rg_output_file_name = (char **)calloc(retval->output_count, sizeof(char *));
- retval->rg_output_file = (samFile**)calloc(retval->output_count, sizeof(samFile*));
- retval->rg_output_header = (bam_hdr_t**)calloc(retval->output_count, sizeof(bam_hdr_t*));
+ // Prevent calloc(0, size);
+ size_t num = retval->output_count ? retval->output_count : 1;
+ retval->rg_index_file_name = (char **)calloc(num, sizeof(char *));
+ retval->rg_output_file_name = (char **)calloc(num, sizeof(char *));
+ retval->rg_output_file = (samFile**)calloc(num, sizeof(samFile*));
+ retval->rg_output_header = (sam_hdr_t**)calloc(num, sizeof(sam_hdr_t*));
retval->rg_hash = kh_init_c2i();
- if (!retval->rg_output_file_name || !retval->rg_output_file || !retval->rg_output_header || !retval->rg_hash) {
+ if (!retval->rg_output_file_name || !retval->rg_output_file || !retval->rg_output_header ||
+ !retval->rg_hash || !retval->rg_index_file_name) {
print_error_errno("split", "Could not initialise output file array");
cleanup_state(retval, false);
return NULL;
&opts->ga.out);
if ( output_filename == NULL ) {
- print_error("split", "Error expanding output filename format string");
cleanup_state(retval, false);
free(input_base_name);
return NULL;
// Record index in hash
int ret;
khiter_t iter = kh_put_c2i(retval->rg_hash, retval->rg_id[i], &ret);
+ if (ret < 0) {
+ print_error_errno("split", "Couldn't add @RG ID to look-up table");
+ cleanup_state(retval, false);
+ free(input_base_name);
+ return NULL;
+ }
kh_val(retval->rg_hash,iter) = i;
// Set and edit header
- retval->rg_output_header[i] = bam_hdr_dup(retval->merged_input_header);
- if ( !filter_header_rg(retval->rg_output_header[i], retval->rg_id[i], arg_list) ) {
+ retval->rg_output_header[i] = sam_hdr_dup(retval->merged_input_header);
+ if (sam_hdr_remove_except(retval->rg_output_header[i], "RG", "ID", retval->rg_id[i]) ||
+ (!opts->no_pg &&
+ sam_hdr_add_pg(retval->rg_output_header[i], "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))) {
print_error("split", "Could not rewrite header for \"%s\"", output_filename);
cleanup_state(retval, false);
free(input_base_name);
}
free(input_base_name);
+ retval->write_index = opts->ga.write_index;
return retval;
}
print_error_errno("split", "Could not write file header to \"%s\"", state->rg_output_file_name[i]);
return false;
}
+ if (state->write_index) {
+ state->rg_index_file_name[i] = auto_index(state->rg_output_file[i],
+ state->rg_output_file_name[i],
+ state->rg_output_header[i]);
+ if (!state->rg_index_file_name[i]) {
+ print_error_errno("split", "Could not create index for file \"%s\"", state->rg_output_file_name[i]);
+ return false;
+ }
+ }
}
bam1_t* file_read = bam_init1();
}
}
+ if (state->write_index) {
+ for (i = 0; i < state->output_count; i++) {
+ if (sam_idx_save(state->rg_output_file[i]) < 0) {
+ print_error_errno("split", "writing index failed");
+ return false;
+ }
+ free(state->rg_index_file_name[i]);
+ }
+ }
+
return true;
}
int ret = 0;
if (!status) return 0;
- if (status->unaccounted_header) bam_hdr_destroy(status->unaccounted_header);
+ if (status->unaccounted_header) sam_hdr_destroy(status->unaccounted_header);
if (status->unaccounted_file) {
if (sam_close(status->unaccounted_file) < 0 && check_close) {
print_error("split", "Error on closing unaccounted file");
size_t i;
for (i = 0; i < status->output_count; i++) {
if (status->rg_output_header && status->rg_output_header[i])
- bam_hdr_destroy(status->rg_output_header[i]);
+ sam_hdr_destroy(status->rg_output_header[i]);
if (status->rg_output_file && status->rg_output_file[i]) {
if (sam_close(status->rg_output_file[i]) < 0 && check_close) {
print_error("split", "Error on closing output file \"%s\"", status->rg_output_file_name[i]);
if (status->rg_output_file_name) free(status->rg_output_file_name[i]);
}
if (status->merged_input_header)
- bam_hdr_destroy(status->merged_input_header);
+ sam_hdr_destroy(status->merged_input_header);
free(status->rg_output_header);
free(status->rg_output_file);
free(status->rg_output_file_name);
+ free(status->rg_index_file_name);
kh_destroy_c2i(status->rg_hash);
free(status->rg_id);
- free(status);
-
if (status->p.pool)
hts_tpool_destroy(status->p.pool);
+ free(status);
return ret;
}
static void cleanup_opts(parsed_opts_t* opts)
{
if (!opts) return;
- free(opts->merged_input_name);
- free(opts->unaccounted_header_name);
- free(opts->unaccounted_name);
- free(opts->output_format_string);
sam_global_args_free(&opts->ga);
free(opts);
}
int main_split(int argc, char** argv)
{
int ret = 1;
- char *arg_list = stringify_argv(argc+1, argv-1);
+ char *arg_list = NULL;
parsed_opts_t* opts = parse_args(argc, argv);
if (!opts) goto cleanup_opts;
+ if (!opts->no_pg && !(arg_list = stringify_argv(argc+1, argv-1)))
+ goto cleanup_opts;
state_t* status = init(opts, arg_list);
if (!status) goto cleanup_opts;
/* bam_split.c -- split subcommand.
- Copyright (C) 2013-2016 Genome Research Ltd.
+ Copyright (C) 2013-2016,2018-2019 Genome Research Ltd.
Author: Martin Pollard <mp15@sanger.ac.uk>
#include <config.h>
-#include <htslib/sam.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <limits.h>
#include <unistd.h>
#include <regex.h>
+#include <assert.h>
+#include <htslib/sam.h>
#include <htslib/khash.h>
#include <htslib/kstring.h>
#include <htslib/cram.h>
KHASH_MAP_INIT_STR(c2i, int)
struct parsed_opts {
- char* merged_input_name;
- char* unaccounted_header_name;
- char* unaccounted_name;
- char* output_format_string;
+ const char *merged_input_name;
+ const char *unaccounted_header_name;
+ const char *unaccounted_name;
+ const char *output_format_string;
bool verbose;
+ int no_pg;
sam_global_args ga;
};
struct state {
samFile* merged_input_file;
- bam_hdr_t* merged_input_header;
+ sam_hdr_t* merged_input_header;
samFile* unaccounted_file;
- bam_hdr_t* unaccounted_header;
+ sam_hdr_t* unaccounted_header;
size_t output_count;
char** rg_id;
+ char **rg_index_file_name;
char **rg_output_file_name;
samFile** rg_output_file;
- bam_hdr_t** rg_output_header;
+ sam_hdr_t** rg_output_header;
kh_c2i_t* rg_hash;
htsThreadPool p;
+ int write_index;
};
typedef struct state state_t;
static void usage(FILE *write_to)
{
fprintf(write_to,
-"Usage: samtools split [-u <unaccounted.bam>[:<unaccounted_header.sam>]]\n"
+"Usage: samtools split [-u <unaccounted.bam>] [-h <unaccounted_header.sam>]\n"
" [-f <format_string>] [-v] <merged.bam>\n"
"Options:\n"
" -f STRING output filename format string [\"%%*_%%#.%%.\"]\n"
" -u FILE1 put reads with no RG tag or an unrecognised RG tag in FILE1\n"
-" -u FILE1:FILE2 ...and override the header with FILE2\n"
-" -v verbose output\n");
- sam_global_opt_help(write_to, "-....@");
+" -h FILE2 ... and override the header with FILE2 (-u file only)\n"
+" -v verbose output\n"
+" --no-PG do not add a PG line\n");
+ sam_global_opt_help(write_to, "-....@..");
fprintf(write_to,
"\n"
"Format string expansions:\n"
{
if (argc == 1) { usage(samtools_stdout); return NULL; }
- const char* optstring = "vf:u:@:";
- char* delim;
+ const char *optstring = "vf:h:u:@:";
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 0, '@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
while ((opt = getopt_long(argc, argv, optstring, lopts, NULL)) != -1) {
switch (opt) {
case 'f':
- retval->output_format_string = strdup(optarg);
- if (! retval->output_format_string ) { perror("cannot allocate output format string memory"); return NULL; }
+ retval->output_format_string = optarg;
+ break;
+ case 'h':
+ retval->unaccounted_header_name = optarg;
break;
case 'v':
retval->verbose = true;
break;
case 'u':
- retval->unaccounted_name = strdup(optarg);
- if (! retval->unaccounted_name ) { perror("cannot allocate string memory"); return NULL; }
- if ((delim = strchr(retval->unaccounted_name, ':')) != NULL) {
- *delim = '\0';
- retval->unaccounted_header_name = strdup(delim+1);
- if (! retval->unaccounted_header_name ) { perror("cannot allocate string memory"); return NULL; }
- }
+ retval->unaccounted_name = optarg;
+ break;
+ case 1:
+ retval->no_pg = 1;
break;
default:
if (parse_sam_global_opt(opt, optarg, lopts, &retval->ga) == 0) break;
}
}
- if (retval->output_format_string == NULL) retval->output_format_string = strdup("%*_%#.%.");
+ if (retval->output_format_string == NULL) retval->output_format_string = "%*_%#.%.";
argc -= optind;
argv += optind;
return NULL;
}
- retval->merged_input_name = strdup(argv[0]);
- if (! retval->merged_input_name ) { perror("cannot allocate string memory"); return NULL; }
+ retval->merged_input_name = argv[0];
return retval;
}
const char* pointer = format_string;
const char* next;
while ((next = strchr(pointer, '%')) != NULL) {
- kputsn(pointer, next-pointer, &str);
+ if (kputsn(pointer, next-pointer, &str) < 0) goto memfail;
++next;
switch (*next) {
case '%':
- kputc('%', &str);
+ if (kputc('%', &str) < 0) goto memfail;
break;
case '*':
- kputs(basename, &str);
+ if (kputs(basename, &str) < 0) goto memfail;
break;
case '#':
- kputl(rg_idx, &str);
+ if (kputl(rg_idx, &str) < 0) goto memfail;
break;
case '!':
- kputs(rg_id, &str);
+ if (kputs(rg_id, &str) < 0) goto memfail;
break;
case '.':
// Only really need to cope with sam, bam, cram
- if (format->format != unknown_format)
- kputs(hts_format_file_extension(format), &str);
- else
- kputs("bam", &str);
+ if (format->format != unknown_format) {
+ if (kputs(hts_format_file_extension(format), &str) < 0)
+ goto memfail;
+ } else {
+ if (kputs("bam", &str) < 0) goto memfail;
+ }
break;
case '\0':
- // Error is: fprintf(samtools_stderr, "bad format string, trailing %%\n");
- free(str.s);
- return NULL;
+ print_error("split", "Trailing %% in filename format string");
+ goto fail;
default:
// Error is: fprintf(samtools_stderr, "bad format string, unknown format specifier\n");
- free(str.s);
- return NULL;
+ print_error("split", "Unknown specifier %%%c in filename format string", *next);
+ goto fail;
}
pointer = next + 1;
}
- kputs(pointer, &str);
+ if (kputs(pointer, &str) < 0) goto memfail;
return ks_release(&str);
+
+ memfail:
+ print_error_errno("split", "Couldn't build output filename");
+ fail:
+ free(str.s);
+ return NULL;
}
// Parse the header, count the number of RG tags and return a list of their names
-static bool count_RG(bam_hdr_t* hdr, size_t* count, char*** output_name)
+static bool count_RG(sam_hdr_t* hdr, size_t* count, char*** output_name)
{
- if (hdr->l_text < 3 ) {
+ char **names = NULL;
+ kstring_t id_val = KS_INITIALIZE;
+ int i, n_rg = sam_hdr_count_lines(hdr, "RG");
+
+ if (n_rg < 0) {
+ print_error("split", "Failed to get @RG IDs");
*count = 0;
*output_name = NULL;
- return true;
- }
- kstring_t input = { 0, 0, NULL };
- kputsn(hdr->text, hdr->l_text, &input);
-
- //////////////////////////////////////////
- // First stage count number of @RG tags //
- //////////////////////////////////////////
- char* pointer = ks_str(&input);
- size_t n_rg = 0;
- // Guard against rare case where @RG is first header line
- // This shouldn't happen but could where @HD is omitted
- if (pointer[0] == '@' && pointer[1] == 'R' && pointer[2] == 'G' ) {
- ++n_rg;
- pointer += 3;
- }
- char* line;
- while ((line = strstr(pointer, "\n@RG")) != NULL) {
- ++n_rg;
- pointer = line + 1;
- }
-
- //////////////////////////////////
- // Second stage locate @RG ID's //
- //////////////////////////////////
- char** names = (char**)calloc(sizeof(char*), n_rg);
- size_t next = 0;
-
- regex_t rg_finder;
- if (regcomp(&rg_finder, "^@RG.*\tID:([!-)+-<>-~][ !-~]*)(\t.*$|$)", REG_EXTENDED|REG_NEWLINE) != 0) {
- free(input.s);
- free(names);
return false;
}
- regmatch_t* matches = (regmatch_t*)calloc(sizeof(regmatch_t),2);
- int error;
- char* begin = ks_str(&input);
-
- while ((error = regexec(&rg_finder, begin, 2, matches, 0)) == 0) {
- kstring_t str = { 0, 0, NULL };
- kputsn(begin+matches[1].rm_so, matches[1].rm_eo-matches[1].rm_so, &str);
- names[next++] = ks_release(&str);
- begin += matches[0].rm_eo;
+
+ if (n_rg == 0) {
+ *count = 0;
+ *output_name = NULL;
+ return true;
}
- if (error != REG_NOMATCH) {
- // cleanup
- regfree(&rg_finder);
- free(matches);
- free(names);
- free(input.s);
- return false;
+ names = calloc(n_rg, sizeof(names[0]));
+ if (!names) goto memfail;
+
+ for (i = 0; i < n_rg; i++) {
+ if (sam_hdr_find_tag_pos(hdr, "RG", i, "ID", &id_val) < 0) goto memfail;
+ names[i] = ks_release(&id_val);
}
- free(matches);
- // return results
*count = n_rg;
*output_name = names;
- regfree(&rg_finder);
- free(input.s);
return true;
+
+ memfail:
+ print_error_errno("split", "Failed to get @RG IDs");
+ *count = 0;
+ *output_name = NULL;
+ ks_free(&id_val);
+ free(names);
+ return false;
}
-// Filters a header of @RG lines where ID != id_keep
-// TODO: strip @PG's descended from other RGs and their descendants
-static bool filter_header_rg(bam_hdr_t* hdr, const char* id_keep, const char *arg_list)
+static int header_compatible(sam_hdr_t *hdr1, sam_hdr_t *hdr2)
{
- kstring_t str = {0, 0, NULL};
-
- regex_t rg_finder;
-
- if (regcomp(&rg_finder, "^@RG.*\tID:([!-)+-<>-~][ !-~]*)(\t.*$|$)", REG_EXTENDED|REG_NEWLINE) != 0) {
- return false;
+ size_t n;
+ if (sam_hdr_nref(hdr1) != sam_hdr_nref(hdr2)) {
+ print_error("split",
+ "Unaccounted header contains wrong number of references");
+ return -1;
}
-
- // regex vars
- char* header = hdr->text;
- regmatch_t* matches = (regmatch_t*)calloc(sizeof(regmatch_t),2);
- kstring_t found_id = { 0, 0, NULL };
- int error;
-
- while ((error = regexec(&rg_finder, header, 2, matches, 0)) == 0) {
- kputsn(header, matches[0].rm_so, &str); // copy header up until the found RG line
-
- found_id.l = 0;
- kputsn(header+matches[1].rm_so, matches[1].rm_eo-matches[1].rm_so, &found_id); // extract ID
- // if it matches keep keep it, else we can just ignore it
- if (strcmp(ks_str(&found_id), id_keep) == 0) {
- kputsn(header+matches[0].rm_so, (matches[0].rm_eo+1)-matches[0].rm_so, &str);
+ for (n = 0; n < sam_hdr_nref(hdr1); n++) {
+ hts_pos_t h1_len = sam_hdr_tid2len(hdr1, n);
+ hts_pos_t h2_len = sam_hdr_tid2len(hdr2, n);
+ if (h1_len != h2_len) {
+ print_error("split",
+ "Unaccounted header reference %zu \"%s\" is not the same length as in the input file",
+ n + 1, sam_hdr_tid2name(hdr2, n));
+ return -1;
}
- // move pointer forward
- header += matches[0].rm_eo+1;
- }
- // cleanup
- free(found_id.s);
- free(matches);
- regfree(&rg_finder);
- // Did we leave loop because of an error?
- if (error != REG_NOMATCH) {
- return false;
}
-
- // Write remainder of string
- kputs(header, &str);
-
- // Modify header
- hdr->l_text = ks_len(&str);
- free(hdr->text);
- hdr->text = ks_release(&str);
-
- // Add the PG line
- SAM_hdr *sh = sam_hdr_parse_(hdr->text, hdr->l_text);
- if (sam_hdr_add_PG(sh, "samtools",
- "VN", samtools_version(),
- arg_list ? "CL": NULL,
- arg_list ? arg_list : NULL,
- NULL) != 0)
- return -1;
-
- free(hdr->text);
- hdr->text = strdup(sam_hdr_str(sh));
- hdr->l_text = sam_hdr_length(sh);
- if (!hdr->text)
- return false;
- sam_hdr_free(sh);
-
- return true;
+ return 0;
}
// Set the initial state
if (opts->ga.nthreads > 0) {
if (!(retval->p.pool = hts_tpool_init(opts->ga.nthreads))) {
fprintf(samtools_stderr, "Error creating thread pool\n");
+ cleanup_state(retval, false);
return NULL;
}
}
retval->merged_input_file = sam_open_format(opts->merged_input_name, "rb", &opts->ga.in);
if (!retval->merged_input_file) {
print_error_errno("split", "Could not open \"%s\"", opts->merged_input_name);
- free(retval);
+ cleanup_state(retval, false);
return NULL;
}
if (retval->p.pool)
if (retval->unaccounted_header == NULL) {
print_error("split", "Could not read header from \"%s\"", opts->unaccounted_header_name);
cleanup_state(retval, false);
+ sam_close(hdr_load);
return NULL;
}
sam_close(hdr_load);
+ if (header_compatible(retval->merged_input_header,
+ retval->unaccounted_header) != 0) {
+ cleanup_state(retval, false);
+ return NULL;
+ }
} else {
- retval->unaccounted_header = bam_hdr_dup(retval->merged_input_header);
+ retval->unaccounted_header = sam_hdr_dup(retval->merged_input_header);
+ if (!opts->no_pg && sam_hdr_add_pg(retval->unaccounted_header, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error("split", "Could not rewrite header for \"%s\"", opts->unaccounted_name);
+ cleanup_state(retval, false);
+ return NULL;
+ }
}
retval->unaccounted_file = sam_open_format(opts->unaccounted_name, "wb", &opts->ga.out);
// Open output files for RGs
if (!count_RG(retval->merged_input_header, &retval->output_count, &retval->rg_id)) return NULL;
if (opts->verbose) fprintf(samtools_stderr, "@RG's found %zu\n",retval->output_count);
-
- retval->rg_output_file_name = (char **)calloc(retval->output_count, sizeof(char *));
- retval->rg_output_file = (samFile**)calloc(retval->output_count, sizeof(samFile*));
- retval->rg_output_header = (bam_hdr_t**)calloc(retval->output_count, sizeof(bam_hdr_t*));
+ // Prevent calloc(0, size);
+ size_t num = retval->output_count ? retval->output_count : 1;
+ retval->rg_index_file_name = (char **)calloc(num, sizeof(char *));
+ retval->rg_output_file_name = (char **)calloc(num, sizeof(char *));
+ retval->rg_output_file = (samFile**)calloc(num, sizeof(samFile*));
+ retval->rg_output_header = (sam_hdr_t**)calloc(num, sizeof(sam_hdr_t*));
retval->rg_hash = kh_init_c2i();
- if (!retval->rg_output_file_name || !retval->rg_output_file || !retval->rg_output_header || !retval->rg_hash) {
+ if (!retval->rg_output_file_name || !retval->rg_output_file || !retval->rg_output_header ||
+ !retval->rg_hash || !retval->rg_index_file_name) {
print_error_errno("split", "Could not initialise output file array");
cleanup_state(retval, false);
return NULL;
&opts->ga.out);
if ( output_filename == NULL ) {
- print_error("split", "Error expanding output filename format string");
cleanup_state(retval, false);
free(input_base_name);
return NULL;
// Record index in hash
int ret;
khiter_t iter = kh_put_c2i(retval->rg_hash, retval->rg_id[i], &ret);
+ if (ret < 0) {
+ print_error_errno("split", "Couldn't add @RG ID to look-up table");
+ cleanup_state(retval, false);
+ free(input_base_name);
+ return NULL;
+ }
kh_val(retval->rg_hash,iter) = i;
// Set and edit header
- retval->rg_output_header[i] = bam_hdr_dup(retval->merged_input_header);
- if ( !filter_header_rg(retval->rg_output_header[i], retval->rg_id[i], arg_list) ) {
+ retval->rg_output_header[i] = sam_hdr_dup(retval->merged_input_header);
+ if (sam_hdr_remove_except(retval->rg_output_header[i], "RG", "ID", retval->rg_id[i]) ||
+ (!opts->no_pg &&
+ sam_hdr_add_pg(retval->rg_output_header[i], "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL))) {
print_error("split", "Could not rewrite header for \"%s\"", output_filename);
cleanup_state(retval, false);
free(input_base_name);
}
free(input_base_name);
+ retval->write_index = opts->ga.write_index;
return retval;
}
print_error_errno("split", "Could not write file header to \"%s\"", state->rg_output_file_name[i]);
return false;
}
+ if (state->write_index) {
+ state->rg_index_file_name[i] = auto_index(state->rg_output_file[i],
+ state->rg_output_file_name[i],
+ state->rg_output_header[i]);
+ if (!state->rg_index_file_name[i]) {
+ print_error_errno("split", "Could not create index for file \"%s\"", state->rg_output_file_name[i]);
+ return false;
+ }
+ }
}
bam1_t* file_read = bam_init1();
}
}
+ if (state->write_index) {
+ for (i = 0; i < state->output_count; i++) {
+ if (sam_idx_save(state->rg_output_file[i]) < 0) {
+ print_error_errno("split", "writing index failed");
+ return false;
+ }
+ free(state->rg_index_file_name[i]);
+ }
+ }
+
return true;
}
int ret = 0;
if (!status) return 0;
- if (status->unaccounted_header) bam_hdr_destroy(status->unaccounted_header);
+ if (status->unaccounted_header) sam_hdr_destroy(status->unaccounted_header);
if (status->unaccounted_file) {
if (sam_close(status->unaccounted_file) < 0 && check_close) {
print_error("split", "Error on closing unaccounted file");
size_t i;
for (i = 0; i < status->output_count; i++) {
if (status->rg_output_header && status->rg_output_header[i])
- bam_hdr_destroy(status->rg_output_header[i]);
+ sam_hdr_destroy(status->rg_output_header[i]);
if (status->rg_output_file && status->rg_output_file[i]) {
if (sam_close(status->rg_output_file[i]) < 0 && check_close) {
print_error("split", "Error on closing output file \"%s\"", status->rg_output_file_name[i]);
if (status->rg_output_file_name) free(status->rg_output_file_name[i]);
}
if (status->merged_input_header)
- bam_hdr_destroy(status->merged_input_header);
+ sam_hdr_destroy(status->merged_input_header);
free(status->rg_output_header);
free(status->rg_output_file);
free(status->rg_output_file_name);
+ free(status->rg_index_file_name);
kh_destroy_c2i(status->rg_hash);
free(status->rg_id);
- free(status);
-
if (status->p.pool)
hts_tpool_destroy(status->p.pool);
+ free(status);
return ret;
}
static void cleanup_opts(parsed_opts_t* opts)
{
if (!opts) return;
- free(opts->merged_input_name);
- free(opts->unaccounted_header_name);
- free(opts->unaccounted_name);
- free(opts->output_format_string);
sam_global_args_free(&opts->ga);
free(opts);
}
int main_split(int argc, char** argv)
{
int ret = 1;
- char *arg_list = stringify_argv(argc+1, argv-1);
+ char *arg_list = NULL;
parsed_opts_t* opts = parse_args(argc, argv);
if (!opts) goto cleanup_opts;
+ if (!opts->no_pg && !(arg_list = stringify_argv(argc+1, argv-1)))
+ goto cleanup_opts;
state_t* status = init(opts, arg_list);
if (!status) goto cleanup_opts;
/* bam_stat.c -- flagstat subcommand.
- Copyright (C) 2009, 2011, 2013-2015 Genome Research Ltd.
+ Copyright (C) 2009, 2011, 2013-2015, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
if ((c)->flag & BAM_FDUP) ++(s)->n_dup[w]; \
} while (0)
-bam_flagstat_t *bam_flagstat_core(samFile *fp, bam_hdr_t *h)
+bam_flagstat_t *bam_flagstat_core(samFile *fp, sam_hdr_t *h)
{
bam_flagstat_t *s;
bam1_t *b;
return buffer;
}
+static const char *percent_json(char *buffer, long long n, long long total)
+{
+ if (total != 0) sprintf(buffer, "%.2f", (float)n / total * 100.0);
+ else strcpy(buffer, "null");
+ return buffer;
+}
+
static void usage_exit(FILE *fp, int exit_status)
{
fprintf(fp, "Usage: samtools flagstat [options] <in.bam>\n");
- sam_global_opt_help(fp, "-.---@");
+ sam_global_opt_help(fp, "-.---@-.");
+ fprintf(fp, " -O, --");
+ fprintf(fp, "output-fmt FORMAT[,OPT[=VAL]]...\n"
+ " Specify output format (json, tsv)\n");
exit(exit_status);
}
+static void out_fmt_default(bam_flagstat_t *s)
+{
+ char b0[16], b1[16];
+ printf("%lld + %lld in total (QC-passed reads + QC-failed reads)\n", s->n_reads[0], s->n_reads[1]);
+ printf("%lld + %lld secondary\n", s->n_secondary[0], s->n_secondary[1]);
+ printf("%lld + %lld supplementary\n", s->n_supp[0], s->n_supp[1]);
+ printf("%lld + %lld duplicates\n", s->n_dup[0], s->n_dup[1]);
+ printf("%lld + %lld mapped (%s : %s)\n", s->n_mapped[0], s->n_mapped[1], percent(b0, s->n_mapped[0], s->n_reads[0]), percent(b1, s->n_mapped[1], s->n_reads[1]));
+ printf("%lld + %lld paired in sequencing\n", s->n_pair_all[0], s->n_pair_all[1]);
+ printf("%lld + %lld read1\n", s->n_read1[0], s->n_read1[1]);
+ printf("%lld + %lld read2\n", s->n_read2[0], s->n_read2[1]);
+ printf("%lld + %lld properly paired (%s : %s)\n", s->n_pair_good[0], s->n_pair_good[1], percent(b0, s->n_pair_good[0], s->n_pair_all[0]), percent(b1, s->n_pair_good[1], s->n_pair_all[1]));
+ printf("%lld + %lld with itself and mate mapped\n", s->n_pair_map[0], s->n_pair_map[1]);
+ printf("%lld + %lld singletons (%s : %s)\n", s->n_sgltn[0], s->n_sgltn[1], percent(b0, s->n_sgltn[0], s->n_pair_all[0]), percent(b1, s->n_sgltn[1], s->n_pair_all[1]));
+ printf("%lld + %lld with mate mapped to a different chr\n", s->n_diffchr[0], s->n_diffchr[1]);
+ printf("%lld + %lld with mate mapped to a different chr (mapQ>=5)\n", s->n_diffhigh[0], s->n_diffhigh[1]);
+}
+
+static void out_fmt_json(bam_flagstat_t *s) {
+ char b0[16], b1[16];
+ printf("{\n \"QC-passed reads\": { \n"
+ " \"total\": %lld, \n"
+ " \"secondary\": %lld, \n"
+ " \"supplementary\": %lld, \n"
+ " \"duplicates\": %lld, \n"
+ " \"mapped\": %lld, \n"
+ " \"mapped %%\": %s, \n"
+ " \"paired in sequencing\": %lld, \n"
+ " \"read1\": %lld, \n"
+ " \"read2\": %lld, \n"
+ " \"properly paired\": %lld, \n"
+ " \"properly paired %%\": %s, \n"
+ " \"with itself and mate mapped\": %lld, \n"
+ " \"singletons\": %lld, \n"
+ " \"singletons %%\": %s, \n"
+ " \"with mate mapped to a different chr\": %lld, \n"
+ " \"with mate mapped to a different chr (mapQ >= 5)\": %lld \n"
+ " },"
+ "\n \"QC-failed reads\": { \n"
+ " \"total\": %lld, \n"
+ " \"secondary\": %lld, \n"
+ " \"supplementary\": %lld, \n"
+ " \"duplicates\": %lld, \n"
+ " \"mapped\": %lld, \n"
+ " \"mapped %%\": %s, \n"
+ " \"paired in sequencing\": %lld, \n"
+ " \"read1\": %lld, \n"
+ " \"read2\": %lld, \n"
+ " \"properly paired\": %lld, \n"
+ " \"properly paired %%\": %s, \n"
+ " \"with itself and mate mapped\": %lld, \n"
+ " \"singletons\": %lld, \n"
+ " \"singletons %%\": %s, \n"
+ " \"with mate mapped to a different chr\": %lld, \n"
+ " \"with mate mapped to a different chr (mapQ >= 5)\": %lld \n"
+ " }\n"
+ "}\n",
+ s->n_reads[0],
+ s->n_secondary[0],
+ s->n_supp[0],
+ s->n_dup[0],
+ s->n_mapped[0],
+ percent_json(b0, s->n_mapped[0], s->n_reads[0]),
+ s->n_pair_all[0],
+ s->n_read1[0],
+ s->n_read2[0],
+ s->n_pair_good[0],
+ percent_json(b0, s->n_pair_good[0], s->n_pair_all[0]),
+ s->n_pair_map[0],
+ s->n_sgltn[0],
+ percent_json(b0, s->n_sgltn[0], s->n_pair_all[0]),
+ s->n_diffchr[0],
+ s->n_diffhigh[0],
+ s->n_reads[1],
+ s->n_secondary[1],
+ s->n_supp[1],
+ s->n_dup[1],
+ s->n_mapped[1],
+ percent_json(b1, s->n_mapped[1], s->n_reads[1]),
+ s->n_pair_all[1],
+ s->n_read1[1],
+ s->n_read2[1],
+ s->n_pair_good[1],
+ percent_json(b1, s->n_pair_good[1], s->n_pair_all[1]),
+ s->n_pair_map[1],
+ s->n_sgltn[1],
+ percent_json(b1, s->n_sgltn[1], s->n_pair_all[1]),
+ s->n_diffchr[1],
+ s->n_diffhigh[1]
+ );
+}
+
+static void out_fmt_tsv(bam_flagstat_t *s) {
+ char b0[16], b1[16];
+ printf("%lld\t%lld\ttotal (QC-passed reads + QC-failed reads)\n", s->n_reads[0], s->n_reads[1]);
+ printf("%lld\t%lld\tsecondary\n", s->n_secondary[0], s->n_secondary[1]);
+ printf("%lld\t%lld\tsupplementary\n", s->n_supp[0], s->n_supp[1]);
+ printf("%lld\t%lld\tduplicates\n", s->n_dup[0], s->n_dup[1]);
+ printf("%lld\t%lld\tmapped\n", s->n_mapped[0], s->n_mapped[1]);
+ printf("%s\t%s\tmapped %%\n", percent(b0, s->n_mapped[0], s->n_reads[0]), percent(b1, s->n_mapped[1], s->n_reads[1]));
+ printf("%lld\t%lld\tpaired in sequencing\n", s->n_pair_all[0], s->n_pair_all[1]);
+ printf("%lld\t%lld\tread1\n", s->n_read1[0], s->n_read1[1]);
+ printf("%lld\t%lld\tread2\n", s->n_read2[0], s->n_read2[1]);
+ printf("%lld\t%lld\tproperly paired\n", s->n_pair_good[0], s->n_pair_good[1]);
+ printf("%s\t%s\tproperly paired %%\n", percent(b0, s->n_pair_good[0], s->n_pair_all[0]), percent(b1, s->n_pair_good[1], s->n_pair_all[1]));
+ printf("%lld\t%lld\twith itself and mate mapped\n", s->n_pair_map[0], s->n_pair_map[1]);
+ printf("%lld\t%lld\tsingletons\n", s->n_sgltn[0], s->n_sgltn[1]);
+ printf("%s\t%s\tsingletons %%\n", percent(b0, s->n_sgltn[0], s->n_pair_all[0]), percent(b1, s->n_sgltn[1], s->n_pair_all[1]));
+ printf("%lld\t%lld\twith mate mapped to a different chr\n", s->n_diffchr[0], s->n_diffchr[1]);
+ printf("%lld\t%lld\twith mate mapped to a different chr (mapQ>=5)\n", s->n_diffhigh[0], s->n_diffhigh[1]);
+}
+
+/*
+ * Select flagstats output format to print.
+ */
+static void output_fmt(bam_flagstat_t *s, const char *out_fmt)
+{
+ if (strcmp(out_fmt, "json") == 0 || strcmp(out_fmt, "JSON") == 0) {
+ out_fmt_json(s);
+ } else if (strcmp(out_fmt, "tsv") == 0 || strcmp(out_fmt, "TSV") == 0) {
+ out_fmt_tsv(s);
+ } else {
+ out_fmt_default(s);
+ }
+}
+
int bam_flagstat(int argc, char *argv[])
{
samFile *fp;
- bam_hdr_t *header;
+ sam_hdr_t *header;
bam_flagstat_t *s;
- char b0[16], b1[16];
+ const char *out_fmt = "default";
int c;
enum {
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
- SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', '-', '@'),
+ SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', '-', '-', '@'),
{NULL, 0, NULL, 0}
};
- while ((c = getopt_long(argc, argv, "@:", lopts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "@:O:", lopts, NULL)) >= 0) {
switch (c) {
+ case 'O':
+ out_fmt = optarg;
+ break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?':
fprintf(stderr, "Failed to read header for \"%s\"\n", argv[optind]);
return 1;
}
+
s = bam_flagstat_core(fp, header);
- printf("%lld + %lld in total (QC-passed reads + QC-failed reads)\n", s->n_reads[0], s->n_reads[1]);
- printf("%lld + %lld secondary\n", s->n_secondary[0], s->n_secondary[1]);
- printf("%lld + %lld supplementary\n", s->n_supp[0], s->n_supp[1]);
- printf("%lld + %lld duplicates\n", s->n_dup[0], s->n_dup[1]);
- printf("%lld + %lld mapped (%s : %s)\n", s->n_mapped[0], s->n_mapped[1], percent(b0, s->n_mapped[0], s->n_reads[0]), percent(b1, s->n_mapped[1], s->n_reads[1]));
- printf("%lld + %lld paired in sequencing\n", s->n_pair_all[0], s->n_pair_all[1]);
- printf("%lld + %lld read1\n", s->n_read1[0], s->n_read1[1]);
- printf("%lld + %lld read2\n", s->n_read2[0], s->n_read2[1]);
- printf("%lld + %lld properly paired (%s : %s)\n", s->n_pair_good[0], s->n_pair_good[1], percent(b0, s->n_pair_good[0], s->n_pair_all[0]), percent(b1, s->n_pair_good[1], s->n_pair_all[1]));
- printf("%lld + %lld with itself and mate mapped\n", s->n_pair_map[0], s->n_pair_map[1]);
- printf("%lld + %lld singletons (%s : %s)\n", s->n_sgltn[0], s->n_sgltn[1], percent(b0, s->n_sgltn[0], s->n_pair_all[0]), percent(b1, s->n_sgltn[1], s->n_pair_all[1]));
- printf("%lld + %lld with mate mapped to a different chr\n", s->n_diffchr[0], s->n_diffchr[1]);
- printf("%lld + %lld with mate mapped to a different chr (mapQ>=5)\n", s->n_diffhigh[0], s->n_diffhigh[1]);
+ output_fmt(s, out_fmt);
free(s);
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
sam_close(fp);
sam_global_args_free(&ga);
return 0;
/* bam_stat.c -- flagstat subcommand.
- Copyright (C) 2009, 2011, 2013-2015 Genome Research Ltd.
+ Copyright (C) 2009, 2011, 2013-2015, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
if ((c)->flag & BAM_FDUP) ++(s)->n_dup[w]; \
} while (0)
-bam_flagstat_t *bam_flagstat_core(samFile *fp, bam_hdr_t *h)
+bam_flagstat_t *bam_flagstat_core(samFile *fp, sam_hdr_t *h)
{
bam_flagstat_t *s;
bam1_t *b;
return buffer;
}
+static const char *percent_json(char *buffer, long long n, long long total)
+{
+ if (total != 0) sprintf(buffer, "%.2f", (float)n / total * 100.0);
+ else strcpy(buffer, "null");
+ return buffer;
+}
+
static void usage_exit(FILE *fp, int exit_status)
{
fprintf(fp, "Usage: samtools flagstat [options] <in.bam>\n");
- sam_global_opt_help(fp, "-.---@");
+ sam_global_opt_help(fp, "-.---@-.");
+ fprintf(fp, " -O, --");
+ fprintf(fp, "output-fmt FORMAT[,OPT[=VAL]]...\n"
+ " Specify output format (json, tsv)\n");
exit(exit_status);
}
+static void out_fmt_default(bam_flagstat_t *s)
+{
+ char b0[16], b1[16];
+ fprintf(samtools_stdout, "%lld + %lld in total (QC-passed reads + QC-failed reads)\n", s->n_reads[0], s->n_reads[1]);
+ fprintf(samtools_stdout, "%lld + %lld secondary\n", s->n_secondary[0], s->n_secondary[1]);
+ fprintf(samtools_stdout, "%lld + %lld supplementary\n", s->n_supp[0], s->n_supp[1]);
+ fprintf(samtools_stdout, "%lld + %lld duplicates\n", s->n_dup[0], s->n_dup[1]);
+ fprintf(samtools_stdout, "%lld + %lld mapped (%s : %s)\n", s->n_mapped[0], s->n_mapped[1], percent(b0, s->n_mapped[0], s->n_reads[0]), percent(b1, s->n_mapped[1], s->n_reads[1]));
+ fprintf(samtools_stdout, "%lld + %lld paired in sequencing\n", s->n_pair_all[0], s->n_pair_all[1]);
+ fprintf(samtools_stdout, "%lld + %lld read1\n", s->n_read1[0], s->n_read1[1]);
+ fprintf(samtools_stdout, "%lld + %lld read2\n", s->n_read2[0], s->n_read2[1]);
+ fprintf(samtools_stdout, "%lld + %lld properly paired (%s : %s)\n", s->n_pair_good[0], s->n_pair_good[1], percent(b0, s->n_pair_good[0], s->n_pair_all[0]), percent(b1, s->n_pair_good[1], s->n_pair_all[1]));
+ fprintf(samtools_stdout, "%lld + %lld with itself and mate mapped\n", s->n_pair_map[0], s->n_pair_map[1]);
+ fprintf(samtools_stdout, "%lld + %lld singletons (%s : %s)\n", s->n_sgltn[0], s->n_sgltn[1], percent(b0, s->n_sgltn[0], s->n_pair_all[0]), percent(b1, s->n_sgltn[1], s->n_pair_all[1]));
+ fprintf(samtools_stdout, "%lld + %lld with mate mapped to a different chr\n", s->n_diffchr[0], s->n_diffchr[1]);
+ fprintf(samtools_stdout, "%lld + %lld with mate mapped to a different chr (mapQ>=5)\n", s->n_diffhigh[0], s->n_diffhigh[1]);
+}
+
+static void out_fmt_json(bam_flagstat_t *s) {
+ char b0[16], b1[16];
+ fprintf(samtools_stdout, "{\n \"QC-passed reads\": { \n"
+ " \"total\": %lld, \n"
+ " \"secondary\": %lld, \n"
+ " \"supplementary\": %lld, \n"
+ " \"duplicates\": %lld, \n"
+ " \"mapped\": %lld, \n"
+ " \"mapped %%\": %s, \n"
+ " \"paired in sequencing\": %lld, \n"
+ " \"read1\": %lld, \n"
+ " \"read2\": %lld, \n"
+ " \"properly paired\": %lld, \n"
+ " \"properly paired %%\": %s, \n"
+ " \"with itself and mate mapped\": %lld, \n"
+ " \"singletons\": %lld, \n"
+ " \"singletons %%\": %s, \n"
+ " \"with mate mapped to a different chr\": %lld, \n"
+ " \"with mate mapped to a different chr (mapQ >= 5)\": %lld \n"
+ " },"
+ "\n \"QC-failed reads\": { \n"
+ " \"total\": %lld, \n"
+ " \"secondary\": %lld, \n"
+ " \"supplementary\": %lld, \n"
+ " \"duplicates\": %lld, \n"
+ " \"mapped\": %lld, \n"
+ " \"mapped %%\": %s, \n"
+ " \"paired in sequencing\": %lld, \n"
+ " \"read1\": %lld, \n"
+ " \"read2\": %lld, \n"
+ " \"properly paired\": %lld, \n"
+ " \"properly paired %%\": %s, \n"
+ " \"with itself and mate mapped\": %lld, \n"
+ " \"singletons\": %lld, \n"
+ " \"singletons %%\": %s, \n"
+ " \"with mate mapped to a different chr\": %lld, \n"
+ " \"with mate mapped to a different chr (mapQ >= 5)\": %lld \n"
+ " }\n"
+ "}\n",
+ s->n_reads[0],
+ s->n_secondary[0],
+ s->n_supp[0],
+ s->n_dup[0],
+ s->n_mapped[0],
+ percent_json(b0, s->n_mapped[0], s->n_reads[0]),
+ s->n_pair_all[0],
+ s->n_read1[0],
+ s->n_read2[0],
+ s->n_pair_good[0],
+ percent_json(b0, s->n_pair_good[0], s->n_pair_all[0]),
+ s->n_pair_map[0],
+ s->n_sgltn[0],
+ percent_json(b0, s->n_sgltn[0], s->n_pair_all[0]),
+ s->n_diffchr[0],
+ s->n_diffhigh[0],
+ s->n_reads[1],
+ s->n_secondary[1],
+ s->n_supp[1],
+ s->n_dup[1],
+ s->n_mapped[1],
+ percent_json(b1, s->n_mapped[1], s->n_reads[1]),
+ s->n_pair_all[1],
+ s->n_read1[1],
+ s->n_read2[1],
+ s->n_pair_good[1],
+ percent_json(b1, s->n_pair_good[1], s->n_pair_all[1]),
+ s->n_pair_map[1],
+ s->n_sgltn[1],
+ percent_json(b1, s->n_sgltn[1], s->n_pair_all[1]),
+ s->n_diffchr[1],
+ s->n_diffhigh[1]
+ );
+}
+
+static void out_fmt_tsv(bam_flagstat_t *s) {
+ char b0[16], b1[16];
+ fprintf(samtools_stdout, "%lld\t%lld\ttotal (QC-passed reads + QC-failed reads)\n", s->n_reads[0], s->n_reads[1]);
+ fprintf(samtools_stdout, "%lld\t%lld\tsecondary\n", s->n_secondary[0], s->n_secondary[1]);
+ fprintf(samtools_stdout, "%lld\t%lld\tsupplementary\n", s->n_supp[0], s->n_supp[1]);
+ fprintf(samtools_stdout, "%lld\t%lld\tduplicates\n", s->n_dup[0], s->n_dup[1]);
+ fprintf(samtools_stdout, "%lld\t%lld\tmapped\n", s->n_mapped[0], s->n_mapped[1]);
+ fprintf(samtools_stdout, "%s\t%s\tmapped %%\n", percent(b0, s->n_mapped[0], s->n_reads[0]), percent(b1, s->n_mapped[1], s->n_reads[1]));
+ fprintf(samtools_stdout, "%lld\t%lld\tpaired in sequencing\n", s->n_pair_all[0], s->n_pair_all[1]);
+ fprintf(samtools_stdout, "%lld\t%lld\tread1\n", s->n_read1[0], s->n_read1[1]);
+ fprintf(samtools_stdout, "%lld\t%lld\tread2\n", s->n_read2[0], s->n_read2[1]);
+ fprintf(samtools_stdout, "%lld\t%lld\tproperly paired\n", s->n_pair_good[0], s->n_pair_good[1]);
+ fprintf(samtools_stdout, "%s\t%s\tproperly paired %%\n", percent(b0, s->n_pair_good[0], s->n_pair_all[0]), percent(b1, s->n_pair_good[1], s->n_pair_all[1]));
+ fprintf(samtools_stdout, "%lld\t%lld\twith itself and mate mapped\n", s->n_pair_map[0], s->n_pair_map[1]);
+ fprintf(samtools_stdout, "%lld\t%lld\tsingletons\n", s->n_sgltn[0], s->n_sgltn[1]);
+ fprintf(samtools_stdout, "%s\t%s\tsingletons %%\n", percent(b0, s->n_sgltn[0], s->n_pair_all[0]), percent(b1, s->n_sgltn[1], s->n_pair_all[1]));
+ fprintf(samtools_stdout, "%lld\t%lld\twith mate mapped to a different chr\n", s->n_diffchr[0], s->n_diffchr[1]);
+ fprintf(samtools_stdout, "%lld\t%lld\twith mate mapped to a different chr (mapQ>=5)\n", s->n_diffhigh[0], s->n_diffhigh[1]);
+}
+
+/*
+ * Select flagstats output format to print.
+ */
+static void output_fmt(bam_flagstat_t *s, const char *out_fmt)
+{
+ if (strcmp(out_fmt, "json") == 0 || strcmp(out_fmt, "JSON") == 0) {
+ out_fmt_json(s);
+ } else if (strcmp(out_fmt, "tsv") == 0 || strcmp(out_fmt, "TSV") == 0) {
+ out_fmt_tsv(s);
+ } else {
+ out_fmt_default(s);
+ }
+}
+
int bam_flagstat(int argc, char *argv[])
{
samFile *fp;
- bam_hdr_t *header;
+ sam_hdr_t *header;
bam_flagstat_t *s;
- char b0[16], b1[16];
+ const char *out_fmt = "default";
int c;
enum {
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
- SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', '-', '@'),
+ SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', '-', '-', '@'),
{NULL, 0, NULL, 0}
};
- while ((c = getopt_long(argc, argv, "@:", lopts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "@:O:", lopts, NULL)) >= 0) {
switch (c) {
+ case 'O':
+ out_fmt = optarg;
+ break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?':
fprintf(samtools_stderr, "Failed to read header for \"%s\"\n", argv[optind]);
return 1;
}
+
s = bam_flagstat_core(fp, header);
- fprintf(samtools_stdout, "%lld + %lld in total (QC-passed reads + QC-failed reads)\n", s->n_reads[0], s->n_reads[1]);
- fprintf(samtools_stdout, "%lld + %lld secondary\n", s->n_secondary[0], s->n_secondary[1]);
- fprintf(samtools_stdout, "%lld + %lld supplementary\n", s->n_supp[0], s->n_supp[1]);
- fprintf(samtools_stdout, "%lld + %lld duplicates\n", s->n_dup[0], s->n_dup[1]);
- fprintf(samtools_stdout, "%lld + %lld mapped (%s : %s)\n", s->n_mapped[0], s->n_mapped[1], percent(b0, s->n_mapped[0], s->n_reads[0]), percent(b1, s->n_mapped[1], s->n_reads[1]));
- fprintf(samtools_stdout, "%lld + %lld paired in sequencing\n", s->n_pair_all[0], s->n_pair_all[1]);
- fprintf(samtools_stdout, "%lld + %lld read1\n", s->n_read1[0], s->n_read1[1]);
- fprintf(samtools_stdout, "%lld + %lld read2\n", s->n_read2[0], s->n_read2[1]);
- fprintf(samtools_stdout, "%lld + %lld properly paired (%s : %s)\n", s->n_pair_good[0], s->n_pair_good[1], percent(b0, s->n_pair_good[0], s->n_pair_all[0]), percent(b1, s->n_pair_good[1], s->n_pair_all[1]));
- fprintf(samtools_stdout, "%lld + %lld with itself and mate mapped\n", s->n_pair_map[0], s->n_pair_map[1]);
- fprintf(samtools_stdout, "%lld + %lld singletons (%s : %s)\n", s->n_sgltn[0], s->n_sgltn[1], percent(b0, s->n_sgltn[0], s->n_pair_all[0]), percent(b1, s->n_sgltn[1], s->n_pair_all[1]));
- fprintf(samtools_stdout, "%lld + %lld with mate mapped to a different chr\n", s->n_diffchr[0], s->n_diffchr[1]);
- fprintf(samtools_stdout, "%lld + %lld with mate mapped to a different chr (mapQ>=5)\n", s->n_diffhigh[0], s->n_diffhigh[1]);
+ output_fmt(s, out_fmt);
free(s);
- bam_hdr_destroy(header);
+ sam_hdr_destroy(header);
sam_close(fp);
sam_global_args_free(&ga);
return 0;
/* bamshuf.c -- collate subcommand.
Copyright (C) 2012 Broad Institute.
- Copyright (C) 2013, 2015, 2018 Genome Research Ltd.
+ Copyright (C) 2013, 2015-2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
}
-static inline int write_to_bin_file(bam1_t *bam, int64_t *count, samFile **bin_files, char **names, bam_hdr_t *header, int files) {
+static inline int write_to_bin_file(bam1_t *bam, int64_t *count, samFile **bin_files, char **names, sam_hdr_t *header, int files) {
uint32_t x;
x = hash_X31_Wang(bam_get_qname(bam)) % files;
static int bamshuf(const char *fn, int n_files, const char *pre, int clevel,
- int is_stdout, const char *output_file, int fast, int store_max, sam_global_args *ga)
+ int is_stdout, const char *output_file, int fast, int store_max, sam_global_args *ga, char *arg_list, int no_pg)
{
samFile *fp, *fpw = NULL, **fpt = NULL;
char **fnt = NULL, modew[8];
bam1_t *b = NULL;
int i, counter, l, r;
- bam_hdr_t *h = NULL;
+ sam_hdr_t *h = NULL;
int64_t j, max_cnt = 0, *cnt = NULL;
elem_t *a = NULL;
htsThreadPool p = {NULL, 0};
goto fail;
}
- if (sam_hdr_change_HD(h, "SO", "unsorted") != 0) {
- print_error("collate",
- "failed to change sort order header to 'unsorted'\n");
- goto fail;
- }
- if (sam_hdr_change_HD(h, "GO", "query") != 0) {
- print_error("collate",
- "failed to change group order header to 'query'\n");
+ if ((-1 == sam_hdr_update_hd(h, "SO", "unsorted", "GO", "query"))
+ && (-1 == sam_hdr_add_line(h, "HD", "VN", SAM_FORMAT_VERSION, "SO", "unsorted", "GO", "query", NULL))
+ ) {
+ print_error("collate", "failed to update HD line\n");
goto fail;
}
}
if (p.pool) hts_set_opt(fpw, HTS_OPT_THREAD_POOL, &p);
+ if (!no_pg && sam_hdr_add_pg(h, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error("collate", "failed to add PG line to header of \"%s\"", output_file);
+ goto fail;
+ }
+
if (sam_hdr_write(fpw, h) < 0) {
print_error_errno("collate", "Couldn't write header");
goto fail;
goto fail;
}
if (p.pool) hts_set_opt(fp, HTS_OPT_THREAD_POOL, &p);
- bam_hdr_destroy(sam_hdr_read(fp)); // Skip over header
+ sam_hdr_destroy(sam_hdr_read(fp)); // Skip over header
// Slurp in one of the split files
for (j = 0; j < c; ++j) {
}
}
- bam_hdr_destroy(h);
+ sam_hdr_destroy(h);
for (j = 0; j < max_cnt; ++j) bam_destroy1(a[j].b);
free(a); free(fnt); free(cnt);
sam_global_args_free(ga);
fail:
if (fp) sam_close(fp);
if (fpw) sam_close(fpw);
- if (h) bam_hdr_destroy(h);
+ if (h) sam_hdr_destroy(h);
for (i = 0; i < n_files; ++i) {
if (fnt) free(fnt[i]);
if (fpt && fpt[i]) sam_close(fpt[i]);
" -f fast (only primary alignments)\n"
" -r working reads stored (with -f) [%d]\n" // reads_store
" -l INT compression level [%d]\n" // DEF_CLEVEL
- " -n INT number of temporary files [%d]\n", // n_files
+ " -n INT number of temporary files [%d]\n" // n_files
+ " --no-PG do not add a PG line\n",
reads_store, DEF_CLEVEL, n_files);
- sam_global_opt_help(fp, "-....@");
+ sam_global_opt_help(fp, "-....@-.");
fprintf(fp,
" <prefix> is required unless the -o or -O options are used.\n");
int main_bamshuf(int argc, char *argv[])
{
- int c, n_files = 64, clevel = DEF_CLEVEL, is_stdout = 0, is_un = 0, fast_coll = 0, reads_store = 10000, ret, pre_mem = 0;
+ int c, n_files = 64, clevel = DEF_CLEVEL, is_stdout = 0, is_un = 0, fast_coll = 0, reads_store = 10000, ret, pre_mem = 0, no_pg = 0;
const char *output_file = NULL;
- char *prefix = NULL;
+ char *prefix = NULL, *arg_list = NULL;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 0, '@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
case 'o': output_file = optarg; break;
case 'f': fast_coll = 1; break;
case 'r': reads_store = atoi(optarg); break;
+ case 1: no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': return usage(stderr, n_files, reads_store);
if (!prefix) return EXIT_FAILURE;
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("collate", "failed to create arg_list");
+ return 1;
+ }
+
ret = bamshuf(argv[optind], n_files, prefix, clevel, is_stdout,
- output_file, fast_coll, reads_store, &ga);
+ output_file, fast_coll, reads_store, &ga, arg_list, no_pg);
if (pre_mem) free(prefix);
+ free(arg_list);
return ret;
}
/* bamshuf.c -- collate subcommand.
Copyright (C) 2012 Broad Institute.
- Copyright (C) 2013, 2015, 2018 Genome Research Ltd.
+ Copyright (C) 2013, 2015-2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
}
-static inline int write_to_bin_file(bam1_t *bam, int64_t *count, samFile **bin_files, char **names, bam_hdr_t *header, int files) {
+static inline int write_to_bin_file(bam1_t *bam, int64_t *count, samFile **bin_files, char **names, sam_hdr_t *header, int files) {
uint32_t x;
x = hash_X31_Wang(bam_get_qname(bam)) % files;
static int bamshuf(const char *fn, int n_files, const char *pre, int clevel,
- int is_samtools_stdout, const char *output_file, int fast, int store_max, sam_global_args *ga)
+ int is_samtools_stdout, const char *output_file, int fast, int store_max, sam_global_args *ga, char *arg_list, int no_pg)
{
samFile *fp, *fpw = NULL, **fpt = NULL;
char **fnt = NULL, modew[8];
bam1_t *b = NULL;
int i, counter, l, r;
- bam_hdr_t *h = NULL;
+ sam_hdr_t *h = NULL;
int64_t j, max_cnt = 0, *cnt = NULL;
elem_t *a = NULL;
htsThreadPool p = {NULL, 0};
goto fail;
}
- if (sam_hdr_change_HD(h, "SO", "unsorted") != 0) {
- print_error("collate",
- "failed to change sort order header to 'unsorted'\n");
- goto fail;
- }
- if (sam_hdr_change_HD(h, "GO", "query") != 0) {
- print_error("collate",
- "failed to change group order header to 'query'\n");
+ if ((-1 == sam_hdr_update_hd(h, "SO", "unsorted", "GO", "query"))
+ && (-1 == sam_hdr_add_line(h, "HD", "VN", SAM_FORMAT_VERSION, "SO", "unsorted", "GO", "query", NULL))
+ ) {
+ print_error("collate", "failed to update HD line\n");
goto fail;
}
}
if (p.pool) hts_set_opt(fpw, HTS_OPT_THREAD_POOL, &p);
+ if (!no_pg && sam_hdr_add_pg(h, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error("collate", "failed to add PG line to header of \"%s\"", output_file);
+ goto fail;
+ }
+
if (sam_hdr_write(fpw, h) < 0) {
print_error_errno("collate", "Couldn't write header");
goto fail;
goto fail;
}
if (p.pool) hts_set_opt(fp, HTS_OPT_THREAD_POOL, &p);
- bam_hdr_destroy(sam_hdr_read(fp)); // Skip over header
+ sam_hdr_destroy(sam_hdr_read(fp)); // Skip over header
// Slurp in one of the split files
for (j = 0; j < c; ++j) {
}
}
- bam_hdr_destroy(h);
+ sam_hdr_destroy(h);
for (j = 0; j < max_cnt; ++j) bam_destroy1(a[j].b);
free(a); free(fnt); free(cnt);
sam_global_args_free(ga);
fail:
if (fp) sam_close(fp);
if (fpw) sam_close(fpw);
- if (h) bam_hdr_destroy(h);
+ if (h) sam_hdr_destroy(h);
for (i = 0; i < n_files; ++i) {
if (fnt) free(fnt[i]);
if (fpt && fpt[i]) sam_close(fpt[i]);
" -f fast (only primary alignments)\n"
" -r working reads stored (with -f) [%d]\n" // reads_store
" -l INT compression level [%d]\n" // DEF_CLEVEL
- " -n INT number of temporary files [%d]\n", // n_files
+ " -n INT number of temporary files [%d]\n" // n_files
+ " --no-PG do not add a PG line\n",
reads_store, DEF_CLEVEL, n_files);
- sam_global_opt_help(fp, "-....@");
+ sam_global_opt_help(fp, "-....@-.");
fprintf(fp,
" <prefix> is required unless the -o or -O options are used.\n");
int main_bamshuf(int argc, char *argv[])
{
- int c, n_files = 64, clevel = DEF_CLEVEL, is_samtools_stdout = 0, is_un = 0, fast_coll = 0, reads_store = 10000, ret, pre_mem = 0;
+ int c, n_files = 64, clevel = DEF_CLEVEL, is_samtools_stdout = 0, is_un = 0, fast_coll = 0, reads_store = 10000, ret, pre_mem = 0, no_pg = 0;
const char *output_file = NULL;
- char *prefix = NULL;
+ char *prefix = NULL, *arg_list = NULL;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 0, '@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
case 'o': output_file = optarg; break;
case 'f': fast_coll = 1; break;
case 'r': reads_store = atoi(optarg); break;
+ case 1: no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': return usage(samtools_stderr, n_files, reads_store);
if (!prefix) return EXIT_FAILURE;
+ if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("collate", "failed to create arg_list");
+ return 1;
+ }
+
ret = bamshuf(argv[optind], n_files, prefix, clevel, is_samtools_stdout,
- output_file, fast_coll, reads_store, &ga);
+ output_file, fast_coll, reads_store, &ga, arg_list, no_pg);
if (pre_mem) free(prefix);
+ free(arg_list);
return ret;
}
/* bamtk.c -- main samtools command front-end.
- Copyright (C) 2008-2018 Genome Research Ltd.
+ Copyright (C) 2008-2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
int main_phase(int argc, char *argv[]);
int main_cat(int argc, char *argv[]);
int main_depth(int argc, char *argv[]);
+int main_coverage(int argc, char *argv[]);
int main_bam2fq(int argc, char *argv[]);
int main_pad2unpad(int argc, char *argv[]);
int main_bedcov(int argc, char *argv[]);
"\n"
" -- Statistics\n"
" bedcov read depth per BED region\n"
+" coverage alignment depth and percent coverage\n"
" depth compute the depth\n"
" flagstat simple stats\n"
" idxstats BAM index stats\n"
else if (strcmp(argv[1], "merge") == 0) ret = bam_merge(argc-1, argv+1);
else if (strcmp(argv[1], "sort") == 0) ret = bam_sort(argc-1, argv+1);
else if (strcmp(argv[1], "index") == 0) ret = bam_index(argc-1, argv+1);
- else if (strcmp(argv[1], "idxstats") == 0) ret = bam_idxstats(argc-1, argv+1);
+ else if (strcmp(argv[1], "idxstat") == 0 ||
+ strcmp(argv[1], "idxstats") == 0) ret = bam_idxstats(argc-1, argv+1);
else if (strcmp(argv[1], "faidx") == 0) ret = faidx_main(argc-1, argv+1);
else if (strcmp(argv[1], "fqidx") == 0) ret = fqidx_main(argc-1, argv+1);
else if (strcmp(argv[1], "dict") == 0) ret = dict_main(argc-1, argv+1);
else if (strcmp(argv[1], "fixmate") == 0) ret = bam_mating(argc-1, argv+1);
else if (strcmp(argv[1], "rmdup") == 0) ret = bam_rmdup(argc-1, argv+1);
else if (strcmp(argv[1], "markdup") == 0) ret = bam_markdup(argc-1, argv+1);
- else if (strcmp(argv[1], "flagstat") == 0) ret = bam_flagstat(argc-1, argv+1);
+ else if (strcmp(argv[1], "flagstat") == 0 ||
+ strcmp(argv[1], "flagstats") == 0) ret = bam_flagstat(argc-1, argv+1);
else if (strcmp(argv[1], "calmd") == 0) ret = bam_fillmd(argc-1, argv+1);
else if (strcmp(argv[1], "fillmd") == 0) ret = bam_fillmd(argc-1, argv+1);
else if (strcmp(argv[1], "reheader") == 0) ret = main_reheader(argc-1, argv+1);
else if (strcmp(argv[1], "targetcut") == 0) ret = main_cut_target(argc-1, argv+1);
else if (strcmp(argv[1], "phase") == 0) ret = main_phase(argc-1, argv+1);
else if (strcmp(argv[1], "depth") == 0) ret = main_depth(argc-1, argv+1);
+ else if (strcmp(argv[1], "coverage") == 0) ret = main_coverage(argc-1, argv+1);
else if (strcmp(argv[1], "bam2fq") == 0 ||
strcmp(argv[1], "fastq") == 0 ||
strcmp(argv[1], "fasta") == 0) ret = main_bam2fq(argc-1, argv+1);
else if (strcmp(argv[1], "bedcov") == 0) ret = main_bedcov(argc-1, argv+1);
else if (strcmp(argv[1], "bamshuf") == 0) ret = main_bamshuf(argc-1, argv+1);
else if (strcmp(argv[1], "collate") == 0) ret = main_bamshuf(argc-1, argv+1);
- else if (strcmp(argv[1], "stats") == 0) ret = main_stats(argc-1, argv+1);
- else if (strcmp(argv[1], "flags") == 0) ret = main_flags(argc-1, argv+1);
+ else if (strcmp(argv[1], "stat") == 0 ||
+ strcmp(argv[1], "stats") == 0) ret = main_stats(argc-1, argv+1);
+ else if (strcmp(argv[1], "flag") == 0 ||
+ strcmp(argv[1], "flags") == 0) ret = main_flags(argc-1, argv+1);
else if (strcmp(argv[1], "split") == 0) ret = main_split(argc-1, argv+1);
else if (strcmp(argv[1], "quickcheck") == 0) ret = main_quickcheck(argc-1, argv+1);
else if (strcmp(argv[1], "addreplacerg") == 0) ret = main_addreplacerg(argc-1, argv+1);
printf(
"samtools %s\n"
"Using htslib %s\n"
-"Copyright (C) 2018 Genome Research Ltd.\n",
+"Copyright (C) 2019 Genome Research Ltd.\n",
samtools_version(), hts_version());
}
else if (strcmp(argv[1], "--version-only") == 0) {
/* bamtk.c -- main samtools command front-end.
- Copyright (C) 2008-2018 Genome Research Ltd.
+ Copyright (C) 2008-2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
int bam_merge(int argc, char *argv[]);
int bam_index(int argc, char *argv[]);
int bam_sort(int argc, char *argv[]);
-//int bam_tview_main(int argc, char *argv[]);
+int bam_tview_main(int argc, char *argv[]);
int bam_mating(int argc, char *argv[]);
int bam_rmdup(int argc, char *argv[]);
int bam_flagstat(int argc, char *argv[]);
int main_phase(int argc, char *argv[]);
int main_cat(int argc, char *argv[]);
int main_depth(int argc, char *argv[]);
+int main_coverage(int argc, char *argv[]);
int main_bam2fq(int argc, char *argv[]);
int main_pad2unpad(int argc, char *argv[]);
int main_bedcov(int argc, char *argv[]);
"\n"
" -- Statistics\n"
" bedcov read depth per BED region\n"
+" coverage alignment depth and percent coverage\n"
" depth compute the depth\n"
" flagstat simple stats\n"
" idxstats BAM index stats\n"
else if (strcmp(argv[1], "merge") == 0) ret = bam_merge(argc-1, argv+1);
else if (strcmp(argv[1], "sort") == 0) ret = bam_sort(argc-1, argv+1);
else if (strcmp(argv[1], "index") == 0) ret = bam_index(argc-1, argv+1);
- else if (strcmp(argv[1], "idxstats") == 0) ret = bam_idxstats(argc-1, argv+1);
+ else if (strcmp(argv[1], "idxstat") == 0 ||
+ strcmp(argv[1], "idxstats") == 0) ret = bam_idxstats(argc-1, argv+1);
else if (strcmp(argv[1], "faidx") == 0) ret = faidx_main(argc-1, argv+1);
else if (strcmp(argv[1], "fqidx") == 0) ret = fqidx_main(argc-1, argv+1);
else if (strcmp(argv[1], "dict") == 0) ret = dict_main(argc-1, argv+1);
else if (strcmp(argv[1], "fixmate") == 0) ret = bam_mating(argc-1, argv+1);
else if (strcmp(argv[1], "rmdup") == 0) ret = bam_rmdup(argc-1, argv+1);
else if (strcmp(argv[1], "markdup") == 0) ret = bam_markdup(argc-1, argv+1);
- else if (strcmp(argv[1], "flagstat") == 0) ret = bam_flagstat(argc-1, argv+1);
+ else if (strcmp(argv[1], "flagstat") == 0 ||
+ strcmp(argv[1], "flagstats") == 0) ret = bam_flagstat(argc-1, argv+1);
else if (strcmp(argv[1], "calmd") == 0) ret = bam_fillmd(argc-1, argv+1);
else if (strcmp(argv[1], "fillmd") == 0) ret = bam_fillmd(argc-1, argv+1);
else if (strcmp(argv[1], "reheader") == 0) ret = main_reheader(argc-1, argv+1);
else if (strcmp(argv[1], "targetcut") == 0) ret = main_cut_target(argc-1, argv+1);
else if (strcmp(argv[1], "phase") == 0) ret = main_phase(argc-1, argv+1);
else if (strcmp(argv[1], "depth") == 0) ret = main_depth(argc-1, argv+1);
+ else if (strcmp(argv[1], "coverage") == 0) ret = main_coverage(argc-1, argv+1);
else if (strcmp(argv[1], "bam2fq") == 0 ||
strcmp(argv[1], "fastq") == 0 ||
strcmp(argv[1], "fasta") == 0) ret = main_bam2fq(argc-1, argv+1);
else if (strcmp(argv[1], "bedcov") == 0) ret = main_bedcov(argc-1, argv+1);
else if (strcmp(argv[1], "bamshuf") == 0) ret = main_bamshuf(argc-1, argv+1);
else if (strcmp(argv[1], "collate") == 0) ret = main_bamshuf(argc-1, argv+1);
- else if (strcmp(argv[1], "stats") == 0) ret = main_stats(argc-1, argv+1);
- else if (strcmp(argv[1], "flags") == 0) ret = main_flags(argc-1, argv+1);
+ else if (strcmp(argv[1], "stat") == 0 ||
+ strcmp(argv[1], "stats") == 0) ret = main_stats(argc-1, argv+1);
+ else if (strcmp(argv[1], "flag") == 0 ||
+ strcmp(argv[1], "flags") == 0) ret = main_flags(argc-1, argv+1);
else if (strcmp(argv[1], "split") == 0) ret = main_split(argc-1, argv+1);
else if (strcmp(argv[1], "quickcheck") == 0) ret = main_quickcheck(argc-1, argv+1);
else if (strcmp(argv[1], "addreplacerg") == 0) ret = main_addreplacerg(argc-1, argv+1);
}
//else if (strcmp(argv[1], "tview") == 0) ret = bam_tview_main(argc-1, argv+1);
else if (strcmp(argv[1], "--version") == 0) {
- fprintf(samtools_stdout,
+ fprintf(samtools_stdout,
"samtools %s\n"
"Using htslib %s\n"
-"Copyright (C) 2018 Genome Research Ltd.\n",
+"Copyright (C) 2019 Genome Research Ltd.\n",
samtools_version(), hts_version());
}
else if (strcmp(argv[1], "--version-only") == 0) {
/* bedcov.c -- bedcov subcommand.
Copyright (C) 2012 Broad Institute.
- Copyright (C) 2013-2014 Genome Research Ltd.
+ Copyright (C) 2013-2014, 2018, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
#include "htslib/kstring.h"
#include "htslib/sam.h"
#include "htslib/thread_pool.h"
+#include "samtools.h"
#include "sam_opts.h"
#include "htslib/kseq.h"
typedef struct {
htsFile *fp;
- bam_hdr_t *header;
+ sam_hdr_t *header;
hts_itr_t *iter;
int min_mapQ;
} aux_t;
int *n_plp, dret, i, j, m, n, c, min_mapQ = 0, skip_DN = 0;
int64_t *cnt;
const bam_pileup1_t **plp;
- int usage = 0;
+ int usage = 0, has_index_file = 0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
{ NULL, 0, NULL, 0 }
};
- while ((c = getopt_long(argc, argv, "Q:j", lopts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "Q:Xj", lopts, NULL)) >= 0) {
switch (c) {
case 'Q': min_mapQ = atoi(optarg); break;
+ case 'X': has_index_file = 1; break;
case 'j': skip_DN = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
fprintf(stderr, "Usage: samtools bedcov [options] <in.bed> <in1.bam> [...]\n\n");
fprintf(stderr, "Options:\n");
fprintf(stderr, " -Q <int> mapping quality threshold [0]\n");
+ fprintf(stderr, " -X use customized index files\n");
fprintf(stderr, " -j do not include deletions (D) and ref skips (N) in bedcov computation\n");
- sam_global_opt_help(stderr, "-.--.-");
+ sam_global_opt_help(stderr, "-.--.--.");
return 1;
}
+ if (has_index_file) {
+ if ((argc - optind - 1) % 2 != 0) { // Calculate # of input BAM files
+ fprintf(stderr, "ERROR: odd number of filenames detected! Each BAM file should have an index file\n");
+ return 1;
+ }
+ n = (argc - optind - 1) / 2;
+ } else {
+ n = argc - optind - 1;
+ }
+
memset(&str, 0, sizeof(kstring_t));
- n = argc - optind - 1;
aux = calloc(n, sizeof(aux_t*));
idx = calloc(n, sizeof(hts_idx_t*));
for (i = 0; i < n; ++i) {
aux[i] = calloc(1, sizeof(aux_t));
aux[i]->min_mapQ = min_mapQ;
aux[i]->fp = sam_open_format(argv[i+optind+1], "r", &ga.in);
- if (aux[i]->fp)
- idx[i] = sam_index_load(aux[i]->fp, argv[i+optind+1]);
+ if (aux[i]->fp) {
+ // If index filename has not been specfied, look in BAM folder
+ if (has_index_file) {
+ idx[i] = sam_index_load2(aux[i]->fp, argv[i+optind+1], argv[i+optind+n+1]);
+ } else {
+ idx[i] = sam_index_load(aux[i]->fp, argv[i+optind+1]);
+ }
+ }
if (aux[i]->fp == 0 || idx[i] == 0) {
fprintf(stderr, "ERROR: fail to open index BAM file '%s'\n", argv[i+optind+1]);
return 2;
cnt = calloc(n, 8);
fp = gzopen(argv[optind], "rb");
+ if (fp == NULL) {
+ print_error_errno("bedcov", "can't open BED file '%s'", argv[optind]);
+ return 2;
+ }
ks = ks_init(fp);
n_plp = calloc(n, sizeof(int));
plp = calloc(n, sizeof(bam_pileup1_t*));
for (i = 0; i < n; ++i) {
if (aux[i]->iter) hts_itr_destroy(aux[i]->iter);
hts_idx_destroy(idx[i]);
- bam_hdr_destroy(aux[i]->header);
+ sam_hdr_destroy(aux[i]->header);
sam_close(aux[i]->fp);
free(aux[i]);
}
/* bedcov.c -- bedcov subcommand.
Copyright (C) 2012 Broad Institute.
- Copyright (C) 2013-2014 Genome Research Ltd.
+ Copyright (C) 2013-2014, 2018, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
#include "htslib/kstring.h"
#include "htslib/sam.h"
#include "htslib/thread_pool.h"
+#include "samtools.h"
#include "sam_opts.h"
#include "htslib/kseq.h"
typedef struct {
htsFile *fp;
- bam_hdr_t *header;
+ sam_hdr_t *header;
hts_itr_t *iter;
int min_mapQ;
} aux_t;
int *n_plp, dret, i, j, m, n, c, min_mapQ = 0, skip_DN = 0;
int64_t *cnt;
const bam_pileup1_t **plp;
- int usage = 0;
+ int usage = 0, has_index_file = 0;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
{ NULL, 0, NULL, 0 }
};
- while ((c = getopt_long(argc, argv, "Q:j", lopts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "Q:Xj", lopts, NULL)) >= 0) {
switch (c) {
case 'Q': min_mapQ = atoi(optarg); break;
+ case 'X': has_index_file = 1; break;
case 'j': skip_DN = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
fprintf(samtools_stderr, "Usage: samtools bedcov [options] <in.bed> <in1.bam> [...]\n\n");
fprintf(samtools_stderr, "Options:\n");
fprintf(samtools_stderr, " -Q <int> mapping quality threshold [0]\n");
+ fprintf(samtools_stderr, " -X use customized index files\n");
fprintf(samtools_stderr, " -j do not include deletions (D) and ref skips (N) in bedcov computation\n");
- sam_global_opt_help(samtools_stderr, "-.--.-");
+ sam_global_opt_help(samtools_stderr, "-.--.--.");
return 1;
}
+ if (has_index_file) {
+ if ((argc - optind - 1) % 2 != 0) { // Calculate # of input BAM files
+ fprintf(samtools_stderr, "ERROR: odd number of filenames detected! Each BAM file should have an index file\n");
+ return 1;
+ }
+ n = (argc - optind - 1) / 2;
+ } else {
+ n = argc - optind - 1;
+ }
+
memset(&str, 0, sizeof(kstring_t));
- n = argc - optind - 1;
aux = calloc(n, sizeof(aux_t*));
idx = calloc(n, sizeof(hts_idx_t*));
for (i = 0; i < n; ++i) {
aux[i] = calloc(1, sizeof(aux_t));
aux[i]->min_mapQ = min_mapQ;
aux[i]->fp = sam_open_format(argv[i+optind+1], "r", &ga.in);
- if (aux[i]->fp)
- idx[i] = sam_index_load(aux[i]->fp, argv[i+optind+1]);
+ if (aux[i]->fp) {
+ // If index filename has not been specfied, look in BAM folder
+ if (has_index_file) {
+ idx[i] = sam_index_load2(aux[i]->fp, argv[i+optind+1], argv[i+optind+n+1]);
+ } else {
+ idx[i] = sam_index_load(aux[i]->fp, argv[i+optind+1]);
+ }
+ }
if (aux[i]->fp == 0 || idx[i] == 0) {
fprintf(samtools_stderr, "ERROR: fail to open index BAM file '%s'\n", argv[i+optind+1]);
return 2;
cnt = calloc(n, 8);
fp = gzopen(argv[optind], "rb");
+ if (fp == NULL) {
+ print_error_errno("bedcov", "can't open BED file '%s'", argv[optind]);
+ return 2;
+ }
ks = ks_init(fp);
n_plp = calloc(n, sizeof(int));
plp = calloc(n, sizeof(bam_pileup1_t*));
for (i = 0; i < n; ++i) {
if (aux[i]->iter) hts_itr_destroy(aux[i]->iter);
hts_idx_destroy(idx[i]);
- bam_hdr_destroy(aux[i]->header);
+ sam_hdr_destroy(aux[i]->header);
sam_close(aux[i]->fp);
free(aux[i]);
}
/* bedidx.c -- BED file indexing.
Copyright (C) 2011 Broad Institute.
- Copyright (C) 2014,2017 Genome Research Ltd.
+ Copyright (C) 2014, 2017-2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
#include "bedidx.h"
#include "htslib/ksort.h"
-KSORT_INIT_GENERIC(uint64_t)
#include "htslib/kseq.h"
KSTREAM_INIT(gzFile, gzread, 8192)
+static inline int lt_pair_pos(hts_pair_pos_t a, hts_pair_pos_t b) {
+ if (a.beg == b.beg) return a.end < b.end;
+ return a.beg < b.beg;
+}
+KSORT_INIT_STATIC(hts_pair_pos_t, hts_pair_pos_t, lt_pair_pos)
+
/*! @typedef
* @abstract bed_reglist_t - value type of the BED hash table
* This structure encodes the list of intervals (ranges) for the regions provided via BED file or
* command line arguments.
- * @field *a pointer to the array of intervals (kept as 64 bit integers). The upper 32 bits
- * encode the beginning of the interval, while the lower 32 bits encode the end, for easy sorting.
- * |-- 32 bits --|-- 32 bits --|
- * |---- beg ----|---- end ----|
+ * @field *a pointer to the array of intervals.
* @field n actual number of elements contained by a
* @field m number of allocated elements to a (n <= m)
* @field *idx index array for computing the minimum offset
*/
typedef struct {
int n, m;
- uint64_t *a;
+ hts_pair_pos_t *a;
int *idx;
int filter;
} bed_reglist_t;
khint_t k;
int i;
const char *reg;
- uint32_t beg, end;
if (!h) {
printf("Hash table is empty!\n");
if ((p = &kh_val(h,k)) != NULL && p->n > 0) {
printf("Filter: %d\n", p->filter);
for (i=0; i<p->n; i++) {
- beg = (uint32_t)(p->a[i]>>32);
- end = (uint32_t)(p->a[i]);
-
- printf("\tinterval[%d]: %d-%d\n",i,beg,end);
+ printf("\tinterval[%d]: %"PRIhts_pos"-%"PRIhts_pos"\n",
+ i,p->a[i].beg,p->a[i].end);
}
} else {
printf("Region '%s' has no intervals!\n", reg);
}
#endif
-static int *bed_index_core(int n, uint64_t *a)
+static int *bed_index_core(int n, hts_pair_pos_t *a)
{
- int i, j, l, *idx;
+ int i, j, l, *idx, *new_idx;
l = 0; idx = 0;
for (i = 0; i < n; ++i) {
- int beg, end;
- beg = a[i]>>32 >> LIDX_SHIFT; end = ((uint32_t)a[i]) >> LIDX_SHIFT;
+ hts_pos_t beg, end;
+ beg = a[i].beg >> LIDX_SHIFT; end = a[i].end >> LIDX_SHIFT;
if (l < end + 1) {
int old_l = l;
l = end + 1;
kroundup32(l);
- idx = realloc(idx, l * sizeof(int));
- if (!idx)
+ new_idx = realloc(idx, l * sizeof(*idx));
+ if (!new_idx) {
+ free(idx);
return NULL;
+ }
+ idx = new_idx;
for (j = old_l; j < l; ++j)
idx[j] = -1;
if (kh_exist(h, k)) {
bed_reglist_t *p = &kh_val(h, k);
if (p->idx) free(p->idx);
- ks_introsort(uint64_t, p->n, p->a);
+ ks_introsort(hts_pair_pos_t, p->n, p->a);
p->idx = bed_index_core(p->n, p->a);
}
}
}
-static int bed_minoff(const bed_reglist_t *p, unsigned int beg, unsigned int end) {
+static int bed_minoff(const bed_reglist_t *p, hts_pos_t beg, hts_pos_t end) {
int i, min_off=0;
if (p && p->idx) {
min_off = (beg>>LIDX_SHIFT >= p->n)? p->idx[p->n-1] : p->idx[beg>>LIDX_SHIFT];
if (min_off < 0) { // TODO: this block can be improved, but speed should not matter too much here
- int n = beg>>LIDX_SHIFT;
+ hts_pos_t n = beg>>LIDX_SHIFT;
if (n > p->n)
n = p->n;
for (i = n - 1; i >= 0; --i)
return min_off;
}
-static int bed_overlap_core(const bed_reglist_t *p, int beg, int end)
+static int bed_overlap_core(const bed_reglist_t *p, hts_pos_t beg, hts_pos_t end)
{
int i, min_off;
if (p->n == 0) return 0;
min_off = bed_minoff(p, beg, end);
for (i = min_off; i < p->n; ++i) {
- if ((int)(p->a[i]>>32) >= end) break; // out of range; no need to proceed
- if ((int32_t)p->a[i] > beg && (int32_t)(p->a[i]>>32) < end)
+ if (p->a[i].beg >= end) break; // out of range; no need to proceed
+ if (p->a[i].end > beg && p->a[i].beg < end)
return 1; // find the overlap; return
}
return 0;
}
-int bed_overlap(const void *_h, const char *chr, int beg, int end)
+int bed_overlap(const void *_h, const char *chr, hts_pos_t beg, hts_pos_t end)
{
const reghash_t *h = (const reghash_t*)_h;
khint_t k;
continue;
for (new_n = 0, j = 1; j < p->n; j++) {
- if ((uint32_t)p->a[new_n] < (uint32_t)(p->a[j]>>32)) {
+ if (p->a[new_n].end < p->a[j].beg) {
p->a[++new_n] = p->a[j];
} else {
- if ((uint32_t)p->a[new_n] < (uint32_t)p->a[j])
- p->a[new_n] = (p->a[new_n] & 0xFFFFFFFF00000000) | (uint32_t)(p->a[j]);
+ if (p->a[new_n].end < p->a[j].end)
+ p->a[new_n].end = p->a[j].end;
}
}
if (fp == 0) return 0;
ks = ks_init(fp);
if (NULL == ks) goto fail; // In case ks_init ever gets error checking...
- while (ks_getuntil(ks, KS_SEP_LINE, &str, &dret) > 0) { // read a line
+ int ks_len;
+ while ((ks_len = ks_getuntil(ks, KS_SEP_LINE, &str, &dret)) >= 0) { // read a line
char *ref = str.s, *ref_end;
- unsigned int beg = 0, end = 0;
+ uint64_t beg = 0, end = 0;
int num = 0;
khint_t k;
bed_reglist_t *p;
+ if (ks_len == 0)
+ continue; // skip blank lines
+
line++;
while (*ref && isspace(*ref)) ref++;
if ('\0' == *ref) continue; // Skip blank lines
while (*ref_end && !isspace(*ref_end)) ref_end++;
if ('\0' != *ref_end) {
*ref_end = '\0'; // terminate ref and look for start, end
- num = sscanf(ref_end + 1, "%u %u", &beg, &end);
+ num = sscanf(ref_end + 1, "%"SCNu64" %"SCNu64, &beg, &end);
}
if (1 == num) { // VCF-style format
end = beg--; // Counts from 1 instead of 0 for BED files
} else {
fprintf(stderr,
"[bed_read] Parse error reading \"%s\" at line %u : "
- "end (%u) must not be less than start (%u)\n",
+ "end (%"PRIu64") must not be less "
+ "than start (%"PRIu64")\n",
fn, line, end, beg);
}
errno = 0; // Prevent caller from printing misleading error messages
// Add begin,end to the list
if (p->n == p->m) {
p->m = p->m ? p->m<<1 : 4;
- p->a = realloc(p->a, p->m * sizeof(uint64_t));
- if (NULL == p->a) goto fail;
+ hts_pair_pos_t *new_a = realloc(p->a, p->m * sizeof(p->a[0]));
+ if (NULL == new_a) goto fail;
+ p->a = new_a;
}
- p->a[p->n++] = (uint64_t)beg<<32 | end;
+ p->a[p->n].beg = beg;
+ p->a[p->n++].end = end;
}
// FIXME: Need to check for errors in ks_getuntil. At the moment it
// doesn't look like it can return one. Possibly use gzgets instead?
+ if (gzclose(fp) != Z_OK) {
+ fp = NULL;
+ goto fail;
+ }
ks_destroy(ks);
- gzclose(fp);
free(str.s);
bed_index(h);
//bed_unify(h);
kh_destroy(reg, h);
}
-static void *bed_insert(void *reg_hash, char *reg, unsigned int beg, unsigned int end) {
+static void *bed_insert(void *reg_hash, char *reg, hts_pos_t beg, hts_pos_t end) {
reghash_t *h;
khint_t k;
// Add beg and end to the list
if (p->n == p->m) {
p->m = p->m ? p->m<<1 : 4;
- p->a = realloc(p->a, p->m * sizeof(uint64_t));
- if (NULL == p->a) goto fail;
+ hts_pair_pos_t *new_a = realloc(p->a, p->m * sizeof(p->a[0]));
+ if (NULL == new_a) goto fail;
+ p->a = new_a;
}
- p->a[p->n++] = (uint64_t)beg<<32 | end;
+ p->a[p->n].beg = beg;
+ p->a[p->n++].end = end;
fail:
return h;
reghash_t *t;
bed_reglist_t *p, *q;
khint_t l, k;
- uint64_t *new_a;
+ hts_pair_pos_t *new_a;
int i, j, new_n, min_off;
const char *reg;
- uint32_t beg, end;
+ hts_pos_t beg, end;
h = (reghash_t *)reg_hash;
t = (reghash_t *)tmp_hash;
if (k == kh_end(h) || !(p = &kh_val(h, k)) || !(p->n))
continue;
- new_a = (uint64_t *)calloc(q->n + p->n, sizeof(uint64_t));
+ new_a = calloc(q->n + p->n, sizeof(new_a[0]));
if (!new_a)
return NULL;
new_n = 0;
for (i = 0; i < q->n; i++) {
- beg = (uint32_t)(q->a[i]>>32);
- end = (uint32_t)(q->a[i]);
+ beg = q->a[i].beg;
+ end = q->a[i].end;
min_off = bed_minoff(p, beg, end);
for (j = min_off; j < p->n; ++j) {
- if ((uint32_t)(p->a[j]>>32) >= end) break; // out of range; no need to proceed
- if ((uint32_t)(p->a[j]) > beg && (uint32_t)(p->a[j]>>32) < end) {
- new_a[new_n++] = ((uint64_t)MAX((uint32_t)(p->a[j]>>32), beg) << 32) | MIN((uint32_t)p->a[j], end);
+ if (p->a[j].beg >= end) break; // out of range; no need to proceed
+ if (p->a[j].end > beg && p->a[j].beg < end) {
+ new_a[new_n].beg = MAX(p->a[j].beg, beg);
+ new_a[new_n++].end = MIN(p->a[j].end, end);
}
}
}
for (i=first; i<last; i++) {
+ // Note, ideally we would call sam_parse_region here, but it's complicated by not
+ // having the sam header known and the likelihood of the bed file containing data for other
+ // references too which we currently just ignore.
+ //
+ // TO DO...
q = hts_parse_reg(regs[i], &beg, &end);
if (q) {
if ((int)(q - regs[i] + 1) > 1024) {
reglist[count].max_end = 0;
for (j = 0; j < p->n; j++) {
- reglist[count].intervals[j].beg = (uint32_t)(p->a[j]>>32);
- reglist[count].intervals[j].end = (uint32_t)(p->a[j]);
+ reglist[count].intervals[j].beg = p->a[j].beg;
+ reglist[count].intervals[j].end = p->a[j].end;
if (reglist[count].intervals[j].end > reglist[count].max_end)
reglist[count].max_end = reglist[count].intervals[j].end;
/* bedidx.c -- BED file indexing.
Copyright (C) 2011 Broad Institute.
- Copyright (C) 2014,2017 Genome Research Ltd.
+ Copyright (C) 2014, 2017-2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
#include "bedidx.h"
#include "htslib/ksort.h"
-KSORT_INIT_GENERIC(uint64_t)
#include "htslib/kseq.h"
KSTREAM_INIT(gzFile, gzread, 8192)
+static inline int lt_pair_pos(hts_pair_pos_t a, hts_pair_pos_t b) {
+ if (a.beg == b.beg) return a.end < b.end;
+ return a.beg < b.beg;
+}
+KSORT_INIT_STATIC(hts_pair_pos_t, hts_pair_pos_t, lt_pair_pos)
+
/*! @typedef
* @abstract bed_reglist_t - value type of the BED hash table
* This structure encodes the list of intervals (ranges) for the regions provided via BED file or
* command line arguments.
- * @field *a pointer to the array of intervals (kept as 64 bit integers). The upper 32 bits
- * encode the beginning of the interval, while the lower 32 bits encode the end, for easy sorting.
- * |-- 32 bits --|-- 32 bits --|
- * |---- beg ----|---- end ----|
+ * @field *a pointer to the array of intervals.
* @field n actual number of elements contained by a
* @field m number of allocated elements to a (n <= m)
* @field *idx index array for computing the minimum offset
*/
typedef struct {
int n, m;
- uint64_t *a;
+ hts_pair_pos_t *a;
int *idx;
int filter;
} bed_reglist_t;
khint_t k;
int i;
const char *reg;
- uint32_t beg, end;
if (!h) {
fprintf(samtools_stdout, "Hash table is empty!\n");
if ((p = &kh_val(h,k)) != NULL && p->n > 0) {
fprintf(samtools_stdout, "Filter: %d\n", p->filter);
for (i=0; i<p->n; i++) {
- beg = (uint32_t)(p->a[i]>>32);
- end = (uint32_t)(p->a[i]);
-
- fprintf(samtools_stdout, "\tinterval[%d]: %d-%d\n",i,beg,end);
+ fprintf(samtools_stdout, "\tinterval[%d]: %"PRIhts_pos"-%"PRIhts_pos"\n",
+ i,p->a[i].beg,p->a[i].end);
}
} else {
fprintf(samtools_stdout, "Region '%s' has no intervals!\n", reg);
}
#endif
-static int *bed_index_core(int n, uint64_t *a)
+static int *bed_index_core(int n, hts_pair_pos_t *a)
{
- int i, j, l, *idx;
+ int i, j, l, *idx, *new_idx;
l = 0; idx = 0;
for (i = 0; i < n; ++i) {
- int beg, end;
- beg = a[i]>>32 >> LIDX_SHIFT; end = ((uint32_t)a[i]) >> LIDX_SHIFT;
+ hts_pos_t beg, end;
+ beg = a[i].beg >> LIDX_SHIFT; end = a[i].end >> LIDX_SHIFT;
if (l < end + 1) {
int old_l = l;
l = end + 1;
kroundup32(l);
- idx = realloc(idx, l * sizeof(int));
- if (!idx)
+ new_idx = realloc(idx, l * sizeof(*idx));
+ if (!new_idx) {
+ free(idx);
return NULL;
+ }
+ idx = new_idx;
for (j = old_l; j < l; ++j)
idx[j] = -1;
if (kh_exist(h, k)) {
bed_reglist_t *p = &kh_val(h, k);
if (p->idx) free(p->idx);
- ks_introsort(uint64_t, p->n, p->a);
+ ks_introsort(hts_pair_pos_t, p->n, p->a);
p->idx = bed_index_core(p->n, p->a);
}
}
}
-static int bed_minoff(const bed_reglist_t *p, unsigned int beg, unsigned int end) {
+static int bed_minoff(const bed_reglist_t *p, hts_pos_t beg, hts_pos_t end) {
int i, min_off=0;
if (p && p->idx) {
min_off = (beg>>LIDX_SHIFT >= p->n)? p->idx[p->n-1] : p->idx[beg>>LIDX_SHIFT];
if (min_off < 0) { // TODO: this block can be improved, but speed should not matter too much here
- int n = beg>>LIDX_SHIFT;
+ hts_pos_t n = beg>>LIDX_SHIFT;
if (n > p->n)
n = p->n;
for (i = n - 1; i >= 0; --i)
return min_off;
}
-static int bed_overlap_core(const bed_reglist_t *p, int beg, int end)
+static int bed_overlap_core(const bed_reglist_t *p, hts_pos_t beg, hts_pos_t end)
{
int i, min_off;
if (p->n == 0) return 0;
min_off = bed_minoff(p, beg, end);
for (i = min_off; i < p->n; ++i) {
- if ((int)(p->a[i]>>32) >= end) break; // out of range; no need to proceed
- if ((int32_t)p->a[i] > beg && (int32_t)(p->a[i]>>32) < end)
+ if (p->a[i].beg >= end) break; // out of range; no need to proceed
+ if (p->a[i].end > beg && p->a[i].beg < end)
return 1; // find the overlap; return
}
return 0;
}
-int bed_overlap(const void *_h, const char *chr, int beg, int end)
+int bed_overlap(const void *_h, const char *chr, hts_pos_t beg, hts_pos_t end)
{
const reghash_t *h = (const reghash_t*)_h;
khint_t k;
continue;
for (new_n = 0, j = 1; j < p->n; j++) {
- if ((uint32_t)p->a[new_n] < (uint32_t)(p->a[j]>>32)) {
+ if (p->a[new_n].end < p->a[j].beg) {
p->a[++new_n] = p->a[j];
} else {
- if ((uint32_t)p->a[new_n] < (uint32_t)p->a[j])
- p->a[new_n] = (p->a[new_n] & 0xFFFFFFFF00000000) | (uint32_t)(p->a[j]);
+ if (p->a[new_n].end < p->a[j].end)
+ p->a[new_n].end = p->a[j].end;
}
}
if (fp == 0) return 0;
ks = ks_init(fp);
if (NULL == ks) goto fail; // In case ks_init ever gets error checking...
- while (ks_getuntil(ks, KS_SEP_LINE, &str, &dret) > 0) { // read a line
+ int ks_len;
+ while ((ks_len = ks_getuntil(ks, KS_SEP_LINE, &str, &dret)) >= 0) { // read a line
char *ref = str.s, *ref_end;
- unsigned int beg = 0, end = 0;
+ uint64_t beg = 0, end = 0;
int num = 0;
khint_t k;
bed_reglist_t *p;
+ if (ks_len == 0)
+ continue; // skip blank lines
+
line++;
while (*ref && isspace(*ref)) ref++;
if ('\0' == *ref) continue; // Skip blank lines
while (*ref_end && !isspace(*ref_end)) ref_end++;
if ('\0' != *ref_end) {
*ref_end = '\0'; // terminate ref and look for start, end
- num = sscanf(ref_end + 1, "%u %u", &beg, &end);
+ num = sscanf(ref_end + 1, "%"SCNu64" %"SCNu64, &beg, &end);
}
if (1 == num) { // VCF-style format
end = beg--; // Counts from 1 instead of 0 for BED files
} else {
fprintf(samtools_stderr,
"[bed_read] Parse error reading \"%s\" at line %u : "
- "end (%u) must not be less than start (%u)\n",
+ "end (%"PRIu64") must not be less "
+ "than start (%"PRIu64")\n",
fn, line, end, beg);
}
errno = 0; // Prevent caller from printing misleading error messages
// Add begin,end to the list
if (p->n == p->m) {
p->m = p->m ? p->m<<1 : 4;
- p->a = realloc(p->a, p->m * sizeof(uint64_t));
- if (NULL == p->a) goto fail;
+ hts_pair_pos_t *new_a = realloc(p->a, p->m * sizeof(p->a[0]));
+ if (NULL == new_a) goto fail;
+ p->a = new_a;
}
- p->a[p->n++] = (uint64_t)beg<<32 | end;
+ p->a[p->n].beg = beg;
+ p->a[p->n++].end = end;
}
// FIXME: Need to check for errors in ks_getuntil. At the moment it
// doesn't look like it can return one. Possibly use gzgets instead?
+ if (gzclose(fp) != Z_OK) {
+ fp = NULL;
+ goto fail;
+ }
ks_destroy(ks);
- gzclose(fp);
free(str.s);
bed_index(h);
//bed_unify(h);
kh_destroy(reg, h);
}
-static void *bed_insert(void *reg_hash, char *reg, unsigned int beg, unsigned int end) {
+static void *bed_insert(void *reg_hash, char *reg, hts_pos_t beg, hts_pos_t end) {
reghash_t *h;
khint_t k;
// Add beg and end to the list
if (p->n == p->m) {
p->m = p->m ? p->m<<1 : 4;
- p->a = realloc(p->a, p->m * sizeof(uint64_t));
- if (NULL == p->a) goto fail;
+ hts_pair_pos_t *new_a = realloc(p->a, p->m * sizeof(p->a[0]));
+ if (NULL == new_a) goto fail;
+ p->a = new_a;
}
- p->a[p->n++] = (uint64_t)beg<<32 | end;
+ p->a[p->n].beg = beg;
+ p->a[p->n++].end = end;
fail:
return h;
reghash_t *t;
bed_reglist_t *p, *q;
khint_t l, k;
- uint64_t *new_a;
+ hts_pair_pos_t *new_a;
int i, j, new_n, min_off;
const char *reg;
- uint32_t beg, end;
+ hts_pos_t beg, end;
h = (reghash_t *)reg_hash;
t = (reghash_t *)tmp_hash;
if (k == kh_end(h) || !(p = &kh_val(h, k)) || !(p->n))
continue;
- new_a = (uint64_t *)calloc(q->n + p->n, sizeof(uint64_t));
+ new_a = calloc(q->n + p->n, sizeof(new_a[0]));
if (!new_a)
return NULL;
new_n = 0;
for (i = 0; i < q->n; i++) {
- beg = (uint32_t)(q->a[i]>>32);
- end = (uint32_t)(q->a[i]);
+ beg = q->a[i].beg;
+ end = q->a[i].end;
min_off = bed_minoff(p, beg, end);
for (j = min_off; j < p->n; ++j) {
- if ((uint32_t)(p->a[j]>>32) >= end) break; // out of range; no need to proceed
- if ((uint32_t)(p->a[j]) > beg && (uint32_t)(p->a[j]>>32) < end) {
- new_a[new_n++] = ((uint64_t)MAX((uint32_t)(p->a[j]>>32), beg) << 32) | MIN((uint32_t)p->a[j], end);
+ if (p->a[j].beg >= end) break; // out of range; no need to proceed
+ if (p->a[j].end > beg && p->a[j].beg < end) {
+ new_a[new_n].beg = MAX(p->a[j].beg, beg);
+ new_a[new_n++].end = MIN(p->a[j].end, end);
}
}
}
for (i=first; i<last; i++) {
+ // Note, ideally we would call sam_parse_region here, but it's complicated by not
+ // having the sam header known and the likelihood of the bed file containing data for other
+ // references too which we currently just ignore.
+ //
+ // TO DO...
q = hts_parse_reg(regs[i], &beg, &end);
if (q) {
if ((int)(q - regs[i] + 1) > 1024) {
reglist[count].max_end = 0;
for (j = 0; j < p->n; j++) {
- reglist[count].intervals[j].beg = (uint32_t)(p->a[j]>>32);
- reglist[count].intervals[j].end = (uint32_t)(p->a[j]);
+ reglist[count].intervals[j].beg = p->a[j].beg;
+ reglist[count].intervals[j].end = p->a[j].end;
if (reglist[count].intervals[j].end > reglist[count].max_end)
reglist[count].max_end = reglist[count].intervals[j].end;
void *bed_read(const char *fn);
void bed_destroy(void *_h);
-int bed_overlap(const void *_h, const char *chr, int beg, int end);
+int bed_overlap(const void *_h, const char *chr, hts_pos_t beg, hts_pos_t end);
void *bed_hash_regions(void *reg_hash, char **regs, int first, int last, int *op);
const char* bed_get(void *reg_hash, int index, int filter);
hts_reglist_t *bed_reglist(void *reg_hash, int filter, int *count_regs);
--- /dev/null
+/* coverage.c -- samtools coverage subcommand
+
+ Copyright (C) 2018,2019 Florian Breitwieser
+ Portions copyright (C) 2019 Genome Research Ltd.
+
+ Author: Florian P Breitwieser <florian.bw@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE. */
+
+/* This program calculates coverage from multiple BAMs
+ * simutaneously, to achieve random access and to use the BED interface.
+ * To compile this program separately, you may:
+ *
+ * gcc -g -O2 -Wall -o bamcov -D_MAIN_BAMCOV coverage.c -lhts -lz
+ */
+
+// C headers
+#include <config.h>
+
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h> // variadic functions
+#include <limits.h> // INT_MAX
+#include <math.h> // round
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <sys/ioctl.h>
+#endif
+
+#include "htslib/sam.h"
+#include "htslib/hts.h"
+#include "samtools.h"
+#include "sam_opts.h"
+
+const char *VERSION = "0.1";
+
+typedef struct { // auxiliary data structure to hold a BAM file
+ samFile *fp; // file handle
+ sam_hdr_t *hdr; // file header
+ hts_itr_t *iter; // iterator to a region - NULL for us by default
+ int min_mapQ; // mapQ filter
+ int min_len; // length filter
+ unsigned int n_reads; // records the number of reads seen in file
+ unsigned int n_selected_reads; // records the number of reads passing filter
+ unsigned long summed_mapQ; // summed mapQ of all reads passing filter
+ int fail_flags;
+ int required_flags;
+} bam_aux_t;
+
+typedef struct { // auxiliary data structure to hold stats on coverage
+ unsigned long long n_covered_bases;
+ unsigned long long summed_coverage;
+ unsigned long long summed_baseQ;
+ unsigned long long summed_mapQ;
+ unsigned int n_reads;
+ unsigned int n_selected_reads;
+ int32_t tid; // chromosome ID, defined by header
+ hts_pos_t beg;
+ hts_pos_t end;
+ int64_t bin_width;
+} stats_aux_t;
+
+#if __STDC_VERSION__ >= 199901L
+#define VERTICAL_LINE "\u2502" // BOX DRAWINGS LIGHT VERTICAL
+
+// UTF8 specifies block characters in eights going from \u2581 (lower one eight block) to \u2588 (full block)
+// https://en.wikipedia.org/wiki/Block_Elements
+// LOWER ONE EIGHTH BLOCK … FULL BLOCK
+static const char *const BLOCK_CHARS8[8] = {"\u2581", "\u2582", "\u2583", "\u2584", "\u2585", "\u2586", "\u2587", "\u2588"};
+// In some terminals / with some fonts not all UTF8 block characters are supported (e.g. Putty). Use only half and full block for those
+static const char *const BLOCK_CHARS2[2] = {"\u2584", "\u2588"};
+
+#else
+
+// Fall back to explicit UTF-8 encodings of the same characters
+#define VERTICAL_LINE "\xE2\x94\x82"
+
+static const char *const BLOCK_CHARS8[8] = {
+ "\xE2\x96\x81", "\xE2\x96\x82", "\xE2\x96\x83", "\xE2\x96\x84",
+ "\xE2\x96\x85", "\xE2\x96\x86", "\xE2\x96\x87", "\xE2\x96\x88" };
+
+static const char *const BLOCK_CHARS2[2] = {"\xE2\x96\x84", "\xE2\x96\x88"};
+
+#endif
+
+// in bam_plcmd.c
+int read_file_list(const char *file_list, int *n, char **argv[]);
+
+static int usage() {
+ fprintf(stdout, "Usage: samtools coverage [options] in1.bam [in2.bam [...]]\n\n"
+ "Input options:\n"
+ " -b, --bam-list FILE list of input BAM filenames, one per line\n"
+ " -l, --min-read-len INT ignore reads shorter than INT bp [0]\n"
+ " -q, --min-MQ INT base quality threshold [0]\n"
+ " -Q, --min-BQ INT mapping quality threshold [0]\n"
+ " --rf <int|str> required flags: skip reads with mask bits unset []\n"
+ " --ff <int|str> filter flags: skip reads with mask bits set \n"
+ " [UNMAP,SECONDARY,QCFAIL,DUP]\n"
+ "Output options:\n"
+ " -m, --histogram show histogram instead of tabular output\n"
+ " -A, --ascii show only ASCII characters in histogram\n"
+ " -o, --output FILE write output to FILE [stdout]\n"
+ " -H, --no-header don't print a header in tabular mode\n"
+ " -w, --n-bins INT number of bins in histogram [terminal width - 40]\n"
+ " -r, --region REG show specified region. Format: chr:start-end. \n"
+ " -h, --help help (this page)\n");
+
+ fprintf(stdout, "\nGeneric options:\n");
+ sam_global_opt_help(stdout, "-.--.--.");
+
+ fprintf(stdout,
+ "\nSee manpage for additional details.\n"
+ " rname Reference name / chromosome\n"
+ " startpos Start position\n"
+ " endpos End position (or sequence length)\n"
+ " numreads Number reads aligned to the region (after filtering)\n"
+ " covbases Number of covered bases with depth >= 1\n"
+ " coverage Proportion of covered bases [0..1]\n"
+ " meandepth Mean depth of coverage\n"
+ " meanbaseq Mean baseQ in covered region\n"
+ " meanmapq Mean mapQ of selected reads\n"
+ );
+
+ return EXIT_SUCCESS;
+}
+
+static char* center_text(char *text, char *buf, int width) {
+ int len = strlen(text);
+ assert(len <= width);
+ int padding = (width - len) / 2;
+ int padding_ex = (width - len) % 2;
+ if (padding >= 1)
+ sprintf(buf, " %*s%*s", len+padding, text, padding-1+padding_ex, " ");
+ else
+ sprintf(buf, "%s", text);
+
+ return buf;
+}
+
+static char* readable_bps(double base_pairs, char *buf) {
+ const char* units[] = {"", "K", "M", "G", "T"};
+ int i = 0;
+ while (base_pairs >= 1000 && i < (sizeof(units)/sizeof(units[0]) - 1)) {
+ base_pairs /= 1000;
+ i++;
+ }
+ sprintf(buf, "%.*f%s", i, base_pairs, units[i]);
+ return buf;
+}
+
+static void set_read_counts(bam_aux_t **data, stats_aux_t *stats, int n_bam_files) {
+ int i;
+ stats->n_reads = 0;
+ stats->n_selected_reads = 0;
+ stats->summed_mapQ = 0;
+ for (i = 0; i < n_bam_files && data[i]; ++i) {
+ stats->n_reads += data[i]->n_reads;
+ stats->n_selected_reads += data[i]->n_selected_reads;
+ stats->summed_mapQ += data[i]->summed_mapQ;
+ data[i]->n_reads = 0;
+ data[i]->n_selected_reads = 0;
+ data[i]->summed_mapQ = 0;
+ }
+}
+
+// read one alignment from one BAM file
+static int read_bam(void *data, bam1_t *b) {
+ bam_aux_t *aux = (bam_aux_t*)data; // data in fact is a pointer to an auxiliary structure
+ int ret;
+ while (1) {
+ if((ret = aux->iter? sam_itr_next(aux->fp, aux->iter, b) : sam_read1(aux->fp, aux->hdr, b)) < 0) break;
+ ++aux->n_reads;
+
+ if ( aux->fail_flags && (b->core.flag & aux->fail_flags) ) continue;
+ if ( aux->required_flags && !(b->core.flag & aux->required_flags) ) continue;
+ if ( b->core.qual < aux->min_mapQ ) continue;
+ if ( aux->min_len && bam_cigar2qlen(b->core.n_cigar, bam_get_cigar(b)) < aux->min_len ) continue;
+ ++aux->n_selected_reads;
+ aux->summed_mapQ += b->core.qual;
+ break;
+ }
+ return ret;
+}
+
+void print_tabular_line(FILE *file_out, const sam_hdr_t *h, const stats_aux_t *stats) {
+ fputs(sam_hdr_tid2name(h, stats->tid), file_out);
+ double region_len = (double) stats->end - stats->beg;
+ fprintf(file_out, "\t%"PRId64"\t%"PRId64"\t%u\t%llu\t%g\t%g\t%.3g\t%.3g\n",
+ stats->beg+1,
+ stats->end,
+ stats->n_selected_reads,
+ stats->n_covered_bases,
+ 100.0 * stats->n_covered_bases / region_len,
+ stats->summed_coverage / region_len,
+ stats->summed_coverage > 0? stats->summed_baseQ/(double) stats->summed_coverage : 0,
+ stats->n_selected_reads > 0? stats->summed_mapQ/(double) stats->n_selected_reads : 0
+ );
+}
+
+void print_hist(FILE *file_out, const sam_hdr_t *h, const stats_aux_t *stats, const uint32_t *hist,
+ const int hist_size, const bool full_utf) {
+ int i, col;
+ bool show_percentiles = false;
+ const int n_rows = 10;
+ const char * const * BLOCK_CHARS = full_utf? BLOCK_CHARS8 : BLOCK_CHARS2;
+ const int blockchar_len = full_utf? 8 : 2;
+ /*
+ if (stats->beg == 0) {
+ stats->end = h->target_len[stats->tid];
+ }
+ */
+ double region_len = stats->end - stats->beg;
+
+ // Calculate histogram that contains percent covered
+ double hist_data[hist_size];
+ double max_val = 0.0;
+ for (i = 0; i < hist_size; ++i) {
+ hist_data[i] = 100 * hist[i] / (double) stats->bin_width;
+ if (hist_data[i] > max_val) max_val = hist_data[i];
+ }
+
+ char buf[30];
+ fprintf(file_out, "%s (%sbp)\n", sam_hdr_tid2name(h, stats->tid), readable_bps(sam_hdr_tid2len(h, stats->tid), buf));
+
+ double row_bin_size = max_val / (double) n_rows;
+ for (i = n_rows-1; i >= 0; --i) {
+ double current_bin = row_bin_size * i;
+ if (show_percentiles) {
+ fprintf(file_out, ">%3i%% ", i*10);
+ } else {
+ fprintf(file_out, ">%7.2f%% ", current_bin);
+ }
+ fprintf(file_out, VERTICAL_LINE);
+ for (col = 0; col < hist_size; ++col) {
+ // get the difference in eights, or halfs when full UTF8 is not supported
+ int cur_val_diff = round(blockchar_len * (hist_data[col] - current_bin) / row_bin_size) - 1;
+ if (cur_val_diff < 0) {
+ fputc(' ', file_out);
+ } else {
+ if (cur_val_diff >= blockchar_len)
+ cur_val_diff = blockchar_len - 1;
+
+ fprintf(file_out, "%s", BLOCK_CHARS[cur_val_diff]);
+ }
+ }
+ fprintf(file_out, VERTICAL_LINE);
+ fputc(' ', file_out);
+ switch (i) {
+ case 9: fprintf(file_out, "Number of reads: %i", stats->n_selected_reads); break;
+ case 8: if (stats->n_reads - stats->n_selected_reads > 0) fprintf(file_out, " (%i filtered)", stats->n_reads - stats->n_selected_reads); break;
+ case 7: fprintf(file_out, "Covered bases: %sbp", readable_bps(stats->n_covered_bases, buf)); break;
+ case 6: fprintf(file_out, "Percent covered: %.4g%%",
+ 100.0 * stats->n_covered_bases / region_len); break;
+ case 5: fprintf(file_out, "Mean coverage: %.3gx",
+ stats->summed_coverage / region_len); break;
+ case 4: fprintf(file_out, "Mean baseQ: %.3g",
+ stats->summed_baseQ/(double) stats->summed_coverage); break;
+ case 3: fprintf(file_out, "Mean mapQ: %.3g",
+ stats->summed_mapQ/(double) stats->n_selected_reads); break;
+ case 1: fprintf(file_out, "Histo bin width: %sbp",
+ readable_bps(stats->bin_width, buf)); break;
+ case 0: fprintf(file_out, "Histo max bin: %.5g%%", max_val); break;
+ };
+ fputc('\n', file_out);
+ }
+
+ // print x axis. Could be made pretty for widths that are not divisible
+ // by 10 by variable spacing of the labels, instead of placing a label every 10 characters
+ char buf2[50];
+ fprintf(file_out, " %s", center_text(readable_bps(stats->beg + 1, buf), buf2, 10));
+ int rest;
+ for (rest = 10; rest < 10*(hist_size/10); rest += 10) {
+ fprintf(file_out, "%s", center_text(readable_bps(stats->beg + stats->bin_width*rest, buf), buf2, 10));
+ }
+ int last_padding = hist_size%10;
+ fprintf(file_out, "%*s%s", last_padding, " ", center_text(readable_bps(stats->end, buf), buf2, 10));
+ fprintf(file_out, "\n");
+}
+
+int main_coverage(int argc, char *argv[]) {
+ int status = EXIT_SUCCESS;
+
+ int ret, tid, pos, i, j;
+
+ int max_depth = 0;
+ int opt_min_baseQ = 0;
+ int opt_min_mapQ = 0;
+ int opt_min_len = 0;
+ int opt_n_bins = 50;
+ bool opt_full_width = true;
+ char *opt_output_file = NULL;
+ bam_aux_t **data = NULL;
+ bam_mplp_t mplp = NULL;
+ const bam_pileup1_t **plp = NULL;
+ uint32_t *hist = NULL;
+ stats_aux_t *stats = NULL;
+ char *opt_reg = 0; // specified region
+ char *opt_file_list = NULL;
+ int n_bam_files = 0;
+ char **fn = NULL;
+ int fail_flags = (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP); // Default fail flags
+ int required_flags = 0;
+
+ int *n_plp = NULL;
+ sam_hdr_t *h = NULL; // BAM header of the 1st input
+
+ bool opt_print_header = true;
+ bool opt_print_tabular = true;
+ bool opt_print_histogram = false;
+ bool *covered_tids = NULL;
+ bool opt_full_utf = true;
+
+ FILE *file_out = stdout;
+
+ sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
+ static const struct option lopts[] = {
+ SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0, '-'),
+ {"rf", required_argument, NULL, 1}, // require flag
+ {"ff", required_argument, NULL, 2}, // filter flag
+ {"incl-flags", required_argument, NULL, 1}, // require flag
+ {"excl-flags", required_argument, NULL, 2}, // filter flag
+ {"bam-list", required_argument, NULL, 'b'},
+ {"min-read-len", required_argument, NULL, 'L'},
+ {"min-MQ", required_argument, NULL, 'q'},
+ {"min-mq", required_argument, NULL, 'q'},
+ {"min-BQ", required_argument, NULL, 'Q'},
+ {"min-bq", required_argument, NULL, 'Q'},
+ {"histogram", no_argument, NULL, 'm'},
+ {"ascii", no_argument, NULL, 'A'},
+ {"output", required_argument, NULL, 'o'},
+ {"no-header", no_argument, NULL, 'H'},
+ {"n-bins", required_argument, NULL, 'w'},
+ {"region", required_argument, NULL, 'r'},
+ {"help", no_argument, NULL, 'h'},
+ { NULL, 0, NULL, 0 }
+ };
+
+ // parse the command line
+ int c;
+ opterr = 0;
+ while ((c = getopt_long(argc, argv, "Ao:L:q:Q:hHw:r:b:m", lopts, NULL)) != -1) {
+ switch (c) {
+ case 1:
+ if ((required_flags = bam_str2flag(optarg)) < 0) {
+ fprintf(stderr,"Could not parse --rf %s\n", optarg); return EXIT_FAILURE;
+ }; break;
+ case 2:
+ if ((fail_flags = bam_str2flag(optarg)) < 0) {
+ fprintf(stderr,"Could not parse --ff %s\n", optarg); return EXIT_FAILURE;
+ }; break;
+ case 'o': opt_output_file = optarg; opt_full_width = false; break;
+ case 'L': opt_min_len = atoi(optarg); break;
+ case 'q': opt_min_baseQ = atoi(optarg); break;
+ case 'Q': opt_min_mapQ = atoi(optarg); break;
+ case 'w': opt_n_bins = atoi(optarg); opt_full_width = false;
+ opt_print_histogram = true; opt_print_tabular = false;
+ break;
+ case 'r': opt_reg = optarg; break; // parsing a region requires a BAM header (strdup unnecessary)
+ case 'b': opt_file_list = optarg; break;
+ case 'm': opt_print_histogram = true; opt_print_tabular = false; break;
+ case 'A': opt_full_utf = false;
+ opt_print_histogram = true; opt_print_tabular = false;
+ break;
+ case 'H': opt_print_header = false; break;
+ case 'h': return usage();
+ default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
+ /* else fall-through */
+ case '?':
+ if (optopt != '?') { // '-?' appeared on command line
+ if (optopt) { // Bad short option
+ print_error("coverage", "invalid option -- '%c'", optopt);
+ } else { // Bad long option
+ // Do our best. There is no good solution to finding
+ // out what the bad option was.
+ // See, e.g. https://stackoverflow.com/questions/2723888/where-does-getopt-long-store-an-unrecognized-option
+ if (optind > 0 && strncmp(argv[optind - 1], "--", 2) == 0) {
+ print_error("coverage", "unrecognised option '%s'",
+ argv[optind - 1]);
+ }
+ }
+ }
+ return usage();
+ }
+ }
+ if (optind == argc && !opt_file_list)
+ return usage();
+
+ // output file provided by user
+ if (opt_output_file != NULL && strcmp(opt_output_file,"-")!=0) {
+ file_out = fopen( opt_output_file, "w" );
+ if (file_out == NULL) {
+ print_error_errno("coverage", "Cannot open \"%s\" for writing.", opt_output_file);
+ return EXIT_FAILURE;
+ }
+ }
+
+ if (opt_n_bins <= 0 || opt_full_width) {
+ // get number of columns of terminal
+ const char* env_columns = getenv("COLUMNS");
+ int columns = 0;
+ if (env_columns == NULL) {
+#ifdef _WIN32
+ CONSOLE_SCREEN_BUFFER_INFO csbi;
+ if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi)) {
+ columns = csbi.srWindow.Right - csbi.srWindow.Left + 1;
+ }
+#else
+ struct winsize w;
+ if (ioctl(2, TIOCGWINSZ, &w) == 0)
+ columns = w.ws_col;
+#endif
+ } else {
+ columns = atoi(env_columns); // atoi(NULL) returns 0
+ }
+
+ if (columns > 60) {
+ opt_n_bins = columns - 40;
+ } else {
+ opt_n_bins = 40;
+ }
+ }
+
+ // setvbuf(file_out, NULL, _IONBF, 0); //turn off buffering
+
+ // Open all BAM files
+ if (opt_file_list) {
+ // Read file names from opt_file_list into argv, and record the number of files in n_bam_files
+ if (read_file_list(opt_file_list, &n_bam_files, &fn)) {
+ print_error_errno("coverage", "Cannot open file list \"%s\".", opt_file_list);
+ return EXIT_FAILURE;
+ }
+ argv = fn;
+ optind = 0;
+ } else {
+ n_bam_files = argc - optind; // the number of BAMs on the command line
+ }
+
+ data = (bam_aux_t **)calloc(n_bam_files, sizeof(bam_aux_t*)); // data[i] for the i-th BAM file
+ if (!data) {
+ print_error("coverage", "Failed to allocate memory");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+
+ for (i = 0; i < n_bam_files; ++i) {
+ int rf;
+ data[i] = (bam_aux_t *) calloc(1, sizeof(bam_aux_t));
+ if (!data[i]) {
+ print_error("coverage", "Failed to allocate memory");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ data[i]->fp = sam_open_format(argv[optind+i], "r", &ga.in); // open BAM
+
+ if (data[i]->fp == NULL) {
+ print_error_errno("coverage", "Could not open \"%s\"", argv[optind+i]);
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ rf = SAM_FLAG | SAM_RNAME | SAM_POS | SAM_MAPQ | SAM_CIGAR | SAM_SEQ;
+ if (opt_min_baseQ) rf |= SAM_QUAL;
+
+ // Set CRAM options on file handle - returns 0 on success
+ if (hts_set_opt(data[i]->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
+ print_error_errno("coverage", "Failed to set CRAM_OPT_REQUIRED_FIELDS value");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ if (hts_set_opt(data[i]->fp, CRAM_OPT_DECODE_MD, 0)) {
+ print_error_errno("coverage", "Failed to set CRAM_OPT_DECODE_MD value");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ data[i]->min_mapQ = opt_min_mapQ; // set the mapQ filter
+ data[i]->min_len = opt_min_len; // set the qlen filter
+ data[i]->hdr = sam_hdr_read(data[i]->fp); // read the BAM header
+ data[i]->fail_flags = fail_flags;
+ data[i]->required_flags = required_flags;
+ if (data[i]->hdr == NULL) {
+ print_error_errno("coverage", "Could not read header for \"%s\"", argv[optind+i]);
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+
+ // Lookup region if specified
+ if (opt_reg) { // if a region is specified
+ hts_idx_t *idx = sam_index_load(data[i]->fp, argv[optind+i]); // load the index
+ if (idx == NULL) {
+ print_error_errno("coverage", "Failed to load index for \"%s\"", argv[optind+i]);
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ data[i]->iter = sam_itr_querys(idx, data[i]->hdr, opt_reg); // set the iterator
+ hts_idx_destroy(idx); // the index is not needed any more; free the memory
+ if (data[i]->iter == NULL) {
+ print_error_errno("coverage", "Failed to parse region \"%s\"", opt_reg);
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ }
+ }
+
+ if (opt_print_tabular && opt_print_header)
+ fputs("#rname\tstartpos\tendpos\tnumreads\tcovbases\tcoverage\tmeandepth\tmeanbaseq\tmeanmapq\n", file_out);
+
+ h = data[0]->hdr; // easy access to the header of the 1st BAM
+ int n_targets = sam_hdr_nref(h);
+ covered_tids = calloc(n_targets, sizeof(bool));
+ stats = calloc(1, sizeof(stats_aux_t));
+ if (!covered_tids || !stats) {
+ print_error("coverage", "Failed to allocate memory");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+
+ int64_t n_bins = opt_n_bins;
+ if (opt_reg) {
+ stats->tid = data[0]->iter->tid;
+ stats->beg = data[0]->iter->beg; // and to the parsed region coordinates
+ stats->end = data[0]->iter->end;
+ if (stats->end == HTS_POS_MAX) {
+ stats->end = sam_hdr_tid2len(h, stats->tid);
+ }
+ if (opt_n_bins > stats->end - stats->beg) {
+ n_bins = stats->end - stats->beg;
+ }
+ stats->bin_width = (stats->end-stats->beg) / n_bins;
+ } else {
+ stats->tid = -1;
+ }
+
+ int64_t current_bin = 0;
+
+ // the core multi-pileup loop
+ mplp = bam_mplp_init(n_bam_files, read_bam, (void**)data); // initialization
+ if (max_depth > 0)
+ bam_mplp_set_maxcnt(mplp, max_depth); // set maximum coverage depth
+ else if (!max_depth)
+ bam_mplp_set_maxcnt(mplp, INT_MAX);
+
+
+ // Extra info for histogram and coverage counting
+ hist = (uint32_t*) calloc(opt_n_bins, sizeof(uint32_t));
+ n_plp = (int*) calloc(n_bam_files, sizeof(int*)); // n_plp[i] is the number of covering reads from the i-th BAM
+ plp = (const bam_pileup1_t**) calloc(n_bam_files, sizeof(bam_pileup1_t*)); // plp[i] points to the array of covering reads (internal in mplp)
+ if (!hist || !n_plp || !plp) {
+ print_error("coverage", "Failed to allocate memory");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ while ((ret=bam_mplp_auto(mplp, &tid, &pos, n_plp, plp)) > 0) { // come to the next covered position
+
+ if (tid != stats->tid) { // Next target sequence
+ if (stats->tid >= 0) { // It's not the first sequence, print results
+ set_read_counts(data, stats, n_bam_files);
+ if (opt_print_histogram) {
+ print_hist(file_out, h, stats, hist, n_bins, opt_full_utf);
+ fputc('\n', file_out);
+ } else if (opt_print_tabular) {
+ print_tabular_line(file_out, h, stats);
+ }
+
+ // reset data
+ memset(stats, 0, sizeof(stats_aux_t));
+ if (opt_print_histogram)
+ memset(hist, 0, n_bins*sizeof(uint32_t));
+ }
+
+ stats->tid = tid;
+ covered_tids[tid] = true;
+ if (!opt_reg)
+ stats->end = sam_hdr_tid2len(h, tid);
+
+ if (opt_print_histogram) {
+ n_bins = opt_n_bins > stats->end-stats->beg? stats->end-stats->beg : opt_n_bins;
+ stats->bin_width = (stats->end-stats->beg) / n_bins;
+ }
+ }
+ if (pos < stats->beg || pos >= stats->end) continue; // out of range; skip
+ if (tid >= n_targets) continue; // diff number of @SQ lines per file?
+
+ if (opt_print_histogram) {
+ current_bin = (pos - stats->beg) / stats->bin_width;
+ }
+
+ bool count_base = false;
+ for (i = 0; i < n_bam_files; ++i) { // base level filters have to go here
+ int depth_at_pos = n_plp[i];
+ for (j = 0; j < n_plp[i]; ++j) {
+ const bam_pileup1_t *p = plp[i] + j; // DON'T modify plp[][] unless you really know
+
+ if (p->is_del || p->is_refskip) --depth_at_pos; // having dels or refskips at tid:pos
+ else if (p->qpos < p->b->core.l_qseq &&
+ bam_get_qual(p->b)[p->qpos] < opt_min_baseQ) --depth_at_pos; // low base quality
+ else
+ stats->summed_baseQ += bam_get_qual(p->b)[p->qpos];
+ }
+ if (depth_at_pos > 0) {
+ count_base = true;
+ stats->summed_coverage += depth_at_pos;
+ }
+ // hist[current_bin] += depth_at_pos; // Add counts to the histogram here to have one based on coverage
+ //fprintf(file_out, "\t%d", n_plp[i] - m); // this the depth to output
+ }
+ if (count_base) {
+ ++(stats->n_covered_bases);
+ if (opt_print_histogram && current_bin < n_bins)
+ ++(hist[current_bin]); // Histogram based on breadth of coverage
+ }
+ }
+
+ if (stats->tid != -1) {
+ set_read_counts(data, stats, n_bam_files);
+ if (opt_print_histogram) {
+ print_hist(file_out, h, stats, hist, n_bins, opt_full_utf);
+ } else if (opt_print_tabular) {
+ print_tabular_line(file_out, h, stats);
+ }
+ }
+
+
+ if (!opt_reg && opt_print_tabular) {
+ memset(stats, 0, sizeof(stats_aux_t));
+ for (i = 0; i < n_targets; ++i) {
+ if (!covered_tids[i]) {
+ stats->tid = i;
+ stats->end = sam_hdr_tid2len(h, i);
+ print_tabular_line(file_out, h, stats);
+ }
+ }
+ }
+
+ if (ret < 0) status = EXIT_FAILURE;
+
+coverage_end:
+ if (n_plp) free(n_plp);
+ if (plp) free(plp);
+ bam_mplp_destroy(mplp);
+
+ if (covered_tids) free(covered_tids);
+ if (hist) free(hist);
+ if (stats) free(stats);
+
+
+ // Close files and free data structures
+ if (!(file_out == stdout || fclose(file_out) == 0)) {
+ if (status == EXIT_SUCCESS) {
+ print_error_errno("coverage", "error on closing \"%s\"",
+ (opt_output_file && strcmp(opt_output_file, "-") != 0?
+ opt_output_file : "stdout"));
+ status = EXIT_FAILURE;
+ }
+ }
+
+ if (data) {
+ for (i = 0; i < n_bam_files && data[i]; ++i) {
+ sam_hdr_destroy(data[i]->hdr);
+ if (data[i]->fp) sam_close(data[i]->fp);
+ hts_itr_destroy(data[i]->iter);
+ free(data[i]);
+ }
+ free(data);
+ }
+
+ if (opt_file_list && fn) {
+ for (i = 0; i < n_bam_files; ++i)
+ free(fn[i]);
+ free(fn);
+ }
+ sam_global_args_free(&ga);
+
+ return status;
+}
+
+#ifdef _MAIN_BAMCOV
+int main(int argc, char *argv[]) {
+ return main_coverage(argc, argv);
+}
+#endif
--- /dev/null
+#include "samtools.pysam.h"
+
+/* coverage.c -- samtools coverage subcommand
+
+ Copyright (C) 2018,2019 Florian Breitwieser
+ Portions copyright (C) 2019 Genome Research Ltd.
+
+ Author: Florian P Breitwieser <florian.bw@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE. */
+
+/* This program calculates coverage from multiple BAMs
+ * simutaneously, to achieve random access and to use the BED interface.
+ * To compile this program separately, you may:
+ *
+ * gcc -g -O2 -Wall -o bamcov -D_MAIN_BAMCOV coverage.c -lhts -lz
+ */
+
+// C headers
+#include <config.h>
+
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h> // variadic functions
+#include <limits.h> // INT_MAX
+#include <math.h> // round
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <sys/ioctl.h>
+#endif
+
+#include "htslib/sam.h"
+#include "htslib/hts.h"
+#include "samtools.h"
+#include "sam_opts.h"
+
+const char *VERSION = "0.1";
+
+typedef struct { // auxiliary data structure to hold a BAM file
+ samFile *fp; // file handle
+ sam_hdr_t *hdr; // file header
+ hts_itr_t *iter; // iterator to a region - NULL for us by default
+ int min_mapQ; // mapQ filter
+ int min_len; // length filter
+ unsigned int n_reads; // records the number of reads seen in file
+ unsigned int n_selected_reads; // records the number of reads passing filter
+ unsigned long summed_mapQ; // summed mapQ of all reads passing filter
+ int fail_flags;
+ int required_flags;
+} bam_aux_t;
+
+typedef struct { // auxiliary data structure to hold stats on coverage
+ unsigned long long n_covered_bases;
+ unsigned long long summed_coverage;
+ unsigned long long summed_baseQ;
+ unsigned long long summed_mapQ;
+ unsigned int n_reads;
+ unsigned int n_selected_reads;
+ int32_t tid; // chromosome ID, defined by header
+ hts_pos_t beg;
+ hts_pos_t end;
+ int64_t bin_width;
+} stats_aux_t;
+
+#if __STDC_VERSION__ >= 199901L
+#define VERTICAL_LINE "\u2502" // BOX DRAWINGS LIGHT VERTICAL
+
+// UTF8 specifies block characters in eights going from \u2581 (lower one eight block) to \u2588 (full block)
+// https://en.wikipedia.org/wiki/Block_Elements
+// LOWER ONE EIGHTH BLOCK … FULL BLOCK
+static const char *const BLOCK_CHARS8[8] = {"\u2581", "\u2582", "\u2583", "\u2584", "\u2585", "\u2586", "\u2587", "\u2588"};
+// In some terminals / with some fonts not all UTF8 block characters are supported (e.g. Putty). Use only half and full block for those
+static const char *const BLOCK_CHARS2[2] = {"\u2584", "\u2588"};
+
+#else
+
+// Fall back to explicit UTF-8 encodings of the same characters
+#define VERTICAL_LINE "\xE2\x94\x82"
+
+static const char *const BLOCK_CHARS8[8] = {
+ "\xE2\x96\x81", "\xE2\x96\x82", "\xE2\x96\x83", "\xE2\x96\x84",
+ "\xE2\x96\x85", "\xE2\x96\x86", "\xE2\x96\x87", "\xE2\x96\x88" };
+
+static const char *const BLOCK_CHARS2[2] = {"\xE2\x96\x84", "\xE2\x96\x88"};
+
+#endif
+
+// in bam_plcmd.c
+int read_file_list(const char *file_list, int *n, char **argv[]);
+
+static int usage() {
+ fprintf(samtools_stdout, "Usage: samtools coverage [options] in1.bam [in2.bam [...]]\n\n"
+ "Input options:\n"
+ " -b, --bam-list FILE list of input BAM filenames, one per line\n"
+ " -l, --min-read-len INT ignore reads shorter than INT bp [0]\n"
+ " -q, --min-MQ INT base quality threshold [0]\n"
+ " -Q, --min-BQ INT mapping quality threshold [0]\n"
+ " --rf <int|str> required flags: skip reads with mask bits unset []\n"
+ " --ff <int|str> filter flags: skip reads with mask bits set \n"
+ " [UNMAP,SECONDARY,QCFAIL,DUP]\n"
+ "Output options:\n"
+ " -m, --histogram show histogram instead of tabular output\n"
+ " -A, --ascii show only ASCII characters in histogram\n"
+ " -o, --output FILE write output to FILE [samtools_stdout]\n"
+ " -H, --no-header don't print a header in tabular mode\n"
+ " -w, --n-bins INT number of bins in histogram [terminal width - 40]\n"
+ " -r, --region REG show specified region. Format: chr:start-end. \n"
+ " -h, --help help (this page)\n");
+
+ fprintf(samtools_stdout, "\nGeneric options:\n");
+ sam_global_opt_help(samtools_stdout, "-.--.--.");
+
+ fprintf(samtools_stdout,
+ "\nSee manpage for additional details.\n"
+ " rname Reference name / chromosome\n"
+ " startpos Start position\n"
+ " endpos End position (or sequence length)\n"
+ " numreads Number reads aligned to the region (after filtering)\n"
+ " covbases Number of covered bases with depth >= 1\n"
+ " coverage Proportion of covered bases [0..1]\n"
+ " meandepth Mean depth of coverage\n"
+ " meanbaseq Mean baseQ in covered region\n"
+ " meanmapq Mean mapQ of selected reads\n"
+ );
+
+ return EXIT_SUCCESS;
+}
+
+static char* center_text(char *text, char *buf, int width) {
+ int len = strlen(text);
+ assert(len <= width);
+ int padding = (width - len) / 2;
+ int padding_ex = (width - len) % 2;
+ if (padding >= 1)
+ sprintf(buf, " %*s%*s", len+padding, text, padding-1+padding_ex, " ");
+ else
+ sprintf(buf, "%s", text);
+
+ return buf;
+}
+
+static char* readable_bps(double base_pairs, char *buf) {
+ const char* units[] = {"", "K", "M", "G", "T"};
+ int i = 0;
+ while (base_pairs >= 1000 && i < (sizeof(units)/sizeof(units[0]) - 1)) {
+ base_pairs /= 1000;
+ i++;
+ }
+ sprintf(buf, "%.*f%s", i, base_pairs, units[i]);
+ return buf;
+}
+
+static void set_read_counts(bam_aux_t **data, stats_aux_t *stats, int n_bam_files) {
+ int i;
+ stats->n_reads = 0;
+ stats->n_selected_reads = 0;
+ stats->summed_mapQ = 0;
+ for (i = 0; i < n_bam_files && data[i]; ++i) {
+ stats->n_reads += data[i]->n_reads;
+ stats->n_selected_reads += data[i]->n_selected_reads;
+ stats->summed_mapQ += data[i]->summed_mapQ;
+ data[i]->n_reads = 0;
+ data[i]->n_selected_reads = 0;
+ data[i]->summed_mapQ = 0;
+ }
+}
+
+// read one alignment from one BAM file
+static int read_bam(void *data, bam1_t *b) {
+ bam_aux_t *aux = (bam_aux_t*)data; // data in fact is a pointer to an auxiliary structure
+ int ret;
+ while (1) {
+ if((ret = aux->iter? sam_itr_next(aux->fp, aux->iter, b) : sam_read1(aux->fp, aux->hdr, b)) < 0) break;
+ ++aux->n_reads;
+
+ if ( aux->fail_flags && (b->core.flag & aux->fail_flags) ) continue;
+ if ( aux->required_flags && !(b->core.flag & aux->required_flags) ) continue;
+ if ( b->core.qual < aux->min_mapQ ) continue;
+ if ( aux->min_len && bam_cigar2qlen(b->core.n_cigar, bam_get_cigar(b)) < aux->min_len ) continue;
+ ++aux->n_selected_reads;
+ aux->summed_mapQ += b->core.qual;
+ break;
+ }
+ return ret;
+}
+
+void print_tabular_line(FILE *file_out, const sam_hdr_t *h, const stats_aux_t *stats) {
+ fputs(sam_hdr_tid2name(h, stats->tid), file_out);
+ double region_len = (double) stats->end - stats->beg;
+ fprintf(file_out, "\t%"PRId64"\t%"PRId64"\t%u\t%llu\t%g\t%g\t%.3g\t%.3g\n",
+ stats->beg+1,
+ stats->end,
+ stats->n_selected_reads,
+ stats->n_covered_bases,
+ 100.0 * stats->n_covered_bases / region_len,
+ stats->summed_coverage / region_len,
+ stats->summed_coverage > 0? stats->summed_baseQ/(double) stats->summed_coverage : 0,
+ stats->n_selected_reads > 0? stats->summed_mapQ/(double) stats->n_selected_reads : 0
+ );
+}
+
+void print_hist(FILE *file_out, const sam_hdr_t *h, const stats_aux_t *stats, const uint32_t *hist,
+ const int hist_size, const bool full_utf) {
+ int i, col;
+ bool show_percentiles = false;
+ const int n_rows = 10;
+ const char * const * BLOCK_CHARS = full_utf? BLOCK_CHARS8 : BLOCK_CHARS2;
+ const int blockchar_len = full_utf? 8 : 2;
+ /*
+ if (stats->beg == 0) {
+ stats->end = h->target_len[stats->tid];
+ }
+ */
+ double region_len = stats->end - stats->beg;
+
+ // Calculate histogram that contains percent covered
+ double hist_data[hist_size];
+ double max_val = 0.0;
+ for (i = 0; i < hist_size; ++i) {
+ hist_data[i] = 100 * hist[i] / (double) stats->bin_width;
+ if (hist_data[i] > max_val) max_val = hist_data[i];
+ }
+
+ char buf[30];
+ fprintf(file_out, "%s (%sbp)\n", sam_hdr_tid2name(h, stats->tid), readable_bps(sam_hdr_tid2len(h, stats->tid), buf));
+
+ double row_bin_size = max_val / (double) n_rows;
+ for (i = n_rows-1; i >= 0; --i) {
+ double current_bin = row_bin_size * i;
+ if (show_percentiles) {
+ fprintf(file_out, ">%3i%% ", i*10);
+ } else {
+ fprintf(file_out, ">%7.2f%% ", current_bin);
+ }
+ fprintf(file_out, VERTICAL_LINE);
+ for (col = 0; col < hist_size; ++col) {
+ // get the difference in eights, or halfs when full UTF8 is not supported
+ int cur_val_diff = round(blockchar_len * (hist_data[col] - current_bin) / row_bin_size) - 1;
+ if (cur_val_diff < 0) {
+ fputc(' ', file_out);
+ } else {
+ if (cur_val_diff >= blockchar_len)
+ cur_val_diff = blockchar_len - 1;
+
+ fprintf(file_out, "%s", BLOCK_CHARS[cur_val_diff]);
+ }
+ }
+ fprintf(file_out, VERTICAL_LINE);
+ fputc(' ', file_out);
+ switch (i) {
+ case 9: fprintf(file_out, "Number of reads: %i", stats->n_selected_reads); break;
+ case 8: if (stats->n_reads - stats->n_selected_reads > 0) fprintf(file_out, " (%i filtered)", stats->n_reads - stats->n_selected_reads); break;
+ case 7: fprintf(file_out, "Covered bases: %sbp", readable_bps(stats->n_covered_bases, buf)); break;
+ case 6: fprintf(file_out, "Percent covered: %.4g%%",
+ 100.0 * stats->n_covered_bases / region_len); break;
+ case 5: fprintf(file_out, "Mean coverage: %.3gx",
+ stats->summed_coverage / region_len); break;
+ case 4: fprintf(file_out, "Mean baseQ: %.3g",
+ stats->summed_baseQ/(double) stats->summed_coverage); break;
+ case 3: fprintf(file_out, "Mean mapQ: %.3g",
+ stats->summed_mapQ/(double) stats->n_selected_reads); break;
+ case 1: fprintf(file_out, "Histo bin width: %sbp",
+ readable_bps(stats->bin_width, buf)); break;
+ case 0: fprintf(file_out, "Histo max bin: %.5g%%", max_val); break;
+ };
+ fputc('\n', file_out);
+ }
+
+ // print x axis. Could be made pretty for widths that are not divisible
+ // by 10 by variable spacing of the labels, instead of placing a label every 10 characters
+ char buf2[50];
+ fprintf(file_out, " %s", center_text(readable_bps(stats->beg + 1, buf), buf2, 10));
+ int rest;
+ for (rest = 10; rest < 10*(hist_size/10); rest += 10) {
+ fprintf(file_out, "%s", center_text(readable_bps(stats->beg + stats->bin_width*rest, buf), buf2, 10));
+ }
+ int last_padding = hist_size%10;
+ fprintf(file_out, "%*s%s", last_padding, " ", center_text(readable_bps(stats->end, buf), buf2, 10));
+ fprintf(file_out, "\n");
+}
+
+int main_coverage(int argc, char *argv[]) {
+ int status = EXIT_SUCCESS;
+
+ int ret, tid, pos, i, j;
+
+ int max_depth = 0;
+ int opt_min_baseQ = 0;
+ int opt_min_mapQ = 0;
+ int opt_min_len = 0;
+ int opt_n_bins = 50;
+ bool opt_full_width = true;
+ char *opt_output_file = NULL;
+ bam_aux_t **data = NULL;
+ bam_mplp_t mplp = NULL;
+ const bam_pileup1_t **plp = NULL;
+ uint32_t *hist = NULL;
+ stats_aux_t *stats = NULL;
+ char *opt_reg = 0; // specified region
+ char *opt_file_list = NULL;
+ int n_bam_files = 0;
+ char **fn = NULL;
+ int fail_flags = (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP); // Default fail flags
+ int required_flags = 0;
+
+ int *n_plp = NULL;
+ sam_hdr_t *h = NULL; // BAM header of the 1st input
+
+ bool opt_print_header = true;
+ bool opt_print_tabular = true;
+ bool opt_print_histogram = false;
+ bool *covered_tids = NULL;
+ bool opt_full_utf = true;
+
+ FILE *file_out = samtools_stdout;
+
+ sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
+ static const struct option lopts[] = {
+ SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0, '-'),
+ {"rf", required_argument, NULL, 1}, // require flag
+ {"ff", required_argument, NULL, 2}, // filter flag
+ {"incl-flags", required_argument, NULL, 1}, // require flag
+ {"excl-flags", required_argument, NULL, 2}, // filter flag
+ {"bam-list", required_argument, NULL, 'b'},
+ {"min-read-len", required_argument, NULL, 'L'},
+ {"min-MQ", required_argument, NULL, 'q'},
+ {"min-mq", required_argument, NULL, 'q'},
+ {"min-BQ", required_argument, NULL, 'Q'},
+ {"min-bq", required_argument, NULL, 'Q'},
+ {"histogram", no_argument, NULL, 'm'},
+ {"ascii", no_argument, NULL, 'A'},
+ {"output", required_argument, NULL, 'o'},
+ {"no-header", no_argument, NULL, 'H'},
+ {"n-bins", required_argument, NULL, 'w'},
+ {"region", required_argument, NULL, 'r'},
+ {"help", no_argument, NULL, 'h'},
+ { NULL, 0, NULL, 0 }
+ };
+
+ // parse the command line
+ int c;
+ opterr = 0;
+ while ((c = getopt_long(argc, argv, "Ao:L:q:Q:hHw:r:b:m", lopts, NULL)) != -1) {
+ switch (c) {
+ case 1:
+ if ((required_flags = bam_str2flag(optarg)) < 0) {
+ fprintf(samtools_stderr,"Could not parse --rf %s\n", optarg); return EXIT_FAILURE;
+ }; break;
+ case 2:
+ if ((fail_flags = bam_str2flag(optarg)) < 0) {
+ fprintf(samtools_stderr,"Could not parse --ff %s\n", optarg); return EXIT_FAILURE;
+ }; break;
+ case 'o': opt_output_file = optarg; opt_full_width = false; break;
+ case 'L': opt_min_len = atoi(optarg); break;
+ case 'q': opt_min_baseQ = atoi(optarg); break;
+ case 'Q': opt_min_mapQ = atoi(optarg); break;
+ case 'w': opt_n_bins = atoi(optarg); opt_full_width = false;
+ opt_print_histogram = true; opt_print_tabular = false;
+ break;
+ case 'r': opt_reg = optarg; break; // parsing a region requires a BAM header (strdup unnecessary)
+ case 'b': opt_file_list = optarg; break;
+ case 'm': opt_print_histogram = true; opt_print_tabular = false; break;
+ case 'A': opt_full_utf = false;
+ opt_print_histogram = true; opt_print_tabular = false;
+ break;
+ case 'H': opt_print_header = false; break;
+ case 'h': return usage();
+ default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
+ /* else fall-through */
+ case '?':
+ if (optopt != '?') { // '-?' appeared on command line
+ if (optopt) { // Bad short option
+ print_error("coverage", "invalid option -- '%c'", optopt);
+ } else { // Bad long option
+ // Do our best. There is no good solution to finding
+ // out what the bad option was.
+ // See, e.g. https://stackoverflow.com/questions/2723888/where-does-getopt-long-store-an-unrecognized-option
+ if (optind > 0 && strncmp(argv[optind - 1], "--", 2) == 0) {
+ print_error("coverage", "unrecognised option '%s'",
+ argv[optind - 1]);
+ }
+ }
+ }
+ return usage();
+ }
+ }
+ if (optind == argc && !opt_file_list)
+ return usage();
+
+ // output file provided by user
+ if (opt_output_file != NULL && strcmp(opt_output_file,"-")!=0) {
+ file_out = fopen( opt_output_file, "w" );
+ if (file_out == NULL) {
+ print_error_errno("coverage", "Cannot open \"%s\" for writing.", opt_output_file);
+ return EXIT_FAILURE;
+ }
+ }
+
+ if (opt_n_bins <= 0 || opt_full_width) {
+ // get number of columns of terminal
+ const char* env_columns = getenv("COLUMNS");
+ int columns = 0;
+ if (env_columns == NULL) {
+#ifdef _WIN32
+ CONSOLE_SCREEN_BUFFER_INFO csbi;
+ if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi)) {
+ columns = csbi.srWindow.Right - csbi.srWindow.Left + 1;
+ }
+#else
+ struct winsize w;
+ if (ioctl(2, TIOCGWINSZ, &w) == 0)
+ columns = w.ws_col;
+#endif
+ } else {
+ columns = atoi(env_columns); // atoi(NULL) returns 0
+ }
+
+ if (columns > 60) {
+ opt_n_bins = columns - 40;
+ } else {
+ opt_n_bins = 40;
+ }
+ }
+
+ // setvbuf(file_out, NULL, _IONBF, 0); //turn off buffering
+
+ // Open all BAM files
+ if (opt_file_list) {
+ // Read file names from opt_file_list into argv, and record the number of files in n_bam_files
+ if (read_file_list(opt_file_list, &n_bam_files, &fn)) {
+ print_error_errno("coverage", "Cannot open file list \"%s\".", opt_file_list);
+ return EXIT_FAILURE;
+ }
+ argv = fn;
+ optind = 0;
+ } else {
+ n_bam_files = argc - optind; // the number of BAMs on the command line
+ }
+
+ data = (bam_aux_t **)calloc(n_bam_files, sizeof(bam_aux_t*)); // data[i] for the i-th BAM file
+ if (!data) {
+ print_error("coverage", "Failed to allocate memory");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+
+ for (i = 0; i < n_bam_files; ++i) {
+ int rf;
+ data[i] = (bam_aux_t *) calloc(1, sizeof(bam_aux_t));
+ if (!data[i]) {
+ print_error("coverage", "Failed to allocate memory");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ data[i]->fp = sam_open_format(argv[optind+i], "r", &ga.in); // open BAM
+
+ if (data[i]->fp == NULL) {
+ print_error_errno("coverage", "Could not open \"%s\"", argv[optind+i]);
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ rf = SAM_FLAG | SAM_RNAME | SAM_POS | SAM_MAPQ | SAM_CIGAR | SAM_SEQ;
+ if (opt_min_baseQ) rf |= SAM_QUAL;
+
+ // Set CRAM options on file handle - returns 0 on success
+ if (hts_set_opt(data[i]->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
+ print_error_errno("coverage", "Failed to set CRAM_OPT_REQUIRED_FIELDS value");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ if (hts_set_opt(data[i]->fp, CRAM_OPT_DECODE_MD, 0)) {
+ print_error_errno("coverage", "Failed to set CRAM_OPT_DECODE_MD value");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ data[i]->min_mapQ = opt_min_mapQ; // set the mapQ filter
+ data[i]->min_len = opt_min_len; // set the qlen filter
+ data[i]->hdr = sam_hdr_read(data[i]->fp); // read the BAM header
+ data[i]->fail_flags = fail_flags;
+ data[i]->required_flags = required_flags;
+ if (data[i]->hdr == NULL) {
+ print_error_errno("coverage", "Could not read header for \"%s\"", argv[optind+i]);
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+
+ // Lookup region if specified
+ if (opt_reg) { // if a region is specified
+ hts_idx_t *idx = sam_index_load(data[i]->fp, argv[optind+i]); // load the index
+ if (idx == NULL) {
+ print_error_errno("coverage", "Failed to load index for \"%s\"", argv[optind+i]);
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ data[i]->iter = sam_itr_querys(idx, data[i]->hdr, opt_reg); // set the iterator
+ hts_idx_destroy(idx); // the index is not needed any more; free the memory
+ if (data[i]->iter == NULL) {
+ print_error_errno("coverage", "Failed to parse region \"%s\"", opt_reg);
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ }
+ }
+
+ if (opt_print_tabular && opt_print_header)
+ fputs("#rname\tstartpos\tendpos\tnumreads\tcovbases\tcoverage\tmeandepth\tmeanbaseq\tmeanmapq\n", file_out);
+
+ h = data[0]->hdr; // easy access to the header of the 1st BAM
+ int n_targets = sam_hdr_nref(h);
+ covered_tids = calloc(n_targets, sizeof(bool));
+ stats = calloc(1, sizeof(stats_aux_t));
+ if (!covered_tids || !stats) {
+ print_error("coverage", "Failed to allocate memory");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+
+ int64_t n_bins = opt_n_bins;
+ if (opt_reg) {
+ stats->tid = data[0]->iter->tid;
+ stats->beg = data[0]->iter->beg; // and to the parsed region coordinates
+ stats->end = data[0]->iter->end;
+ if (stats->end == HTS_POS_MAX) {
+ stats->end = sam_hdr_tid2len(h, stats->tid);
+ }
+ if (opt_n_bins > stats->end - stats->beg) {
+ n_bins = stats->end - stats->beg;
+ }
+ stats->bin_width = (stats->end-stats->beg) / n_bins;
+ } else {
+ stats->tid = -1;
+ }
+
+ int64_t current_bin = 0;
+
+ // the core multi-pileup loop
+ mplp = bam_mplp_init(n_bam_files, read_bam, (void**)data); // initialization
+ if (max_depth > 0)
+ bam_mplp_set_maxcnt(mplp, max_depth); // set maximum coverage depth
+ else if (!max_depth)
+ bam_mplp_set_maxcnt(mplp, INT_MAX);
+
+
+ // Extra info for histogram and coverage counting
+ hist = (uint32_t*) calloc(opt_n_bins, sizeof(uint32_t));
+ n_plp = (int*) calloc(n_bam_files, sizeof(int*)); // n_plp[i] is the number of covering reads from the i-th BAM
+ plp = (const bam_pileup1_t**) calloc(n_bam_files, sizeof(bam_pileup1_t*)); // plp[i] points to the array of covering reads (internal in mplp)
+ if (!hist || !n_plp || !plp) {
+ print_error("coverage", "Failed to allocate memory");
+ status = EXIT_FAILURE;
+ goto coverage_end;
+ }
+ while ((ret=bam_mplp_auto(mplp, &tid, &pos, n_plp, plp)) > 0) { // come to the next covered position
+
+ if (tid != stats->tid) { // Next target sequence
+ if (stats->tid >= 0) { // It's not the first sequence, print results
+ set_read_counts(data, stats, n_bam_files);
+ if (opt_print_histogram) {
+ print_hist(file_out, h, stats, hist, n_bins, opt_full_utf);
+ fputc('\n', file_out);
+ } else if (opt_print_tabular) {
+ print_tabular_line(file_out, h, stats);
+ }
+
+ // reset data
+ memset(stats, 0, sizeof(stats_aux_t));
+ if (opt_print_histogram)
+ memset(hist, 0, n_bins*sizeof(uint32_t));
+ }
+
+ stats->tid = tid;
+ covered_tids[tid] = true;
+ if (!opt_reg)
+ stats->end = sam_hdr_tid2len(h, tid);
+
+ if (opt_print_histogram) {
+ n_bins = opt_n_bins > stats->end-stats->beg? stats->end-stats->beg : opt_n_bins;
+ stats->bin_width = (stats->end-stats->beg) / n_bins;
+ }
+ }
+ if (pos < stats->beg || pos >= stats->end) continue; // out of range; skip
+ if (tid >= n_targets) continue; // diff number of @SQ lines per file?
+
+ if (opt_print_histogram) {
+ current_bin = (pos - stats->beg) / stats->bin_width;
+ }
+
+ bool count_base = false;
+ for (i = 0; i < n_bam_files; ++i) { // base level filters have to go here
+ int depth_at_pos = n_plp[i];
+ for (j = 0; j < n_plp[i]; ++j) {
+ const bam_pileup1_t *p = plp[i] + j; // DON'T modify plp[][] unless you really know
+
+ if (p->is_del || p->is_refskip) --depth_at_pos; // having dels or refskips at tid:pos
+ else if (p->qpos < p->b->core.l_qseq &&
+ bam_get_qual(p->b)[p->qpos] < opt_min_baseQ) --depth_at_pos; // low base quality
+ else
+ stats->summed_baseQ += bam_get_qual(p->b)[p->qpos];
+ }
+ if (depth_at_pos > 0) {
+ count_base = true;
+ stats->summed_coverage += depth_at_pos;
+ }
+ // hist[current_bin] += depth_at_pos; // Add counts to the histogram here to have one based on coverage
+ //fprintf(file_out, "\t%d", n_plp[i] - m); // this the depth to output
+ }
+ if (count_base) {
+ ++(stats->n_covered_bases);
+ if (opt_print_histogram && current_bin < n_bins)
+ ++(hist[current_bin]); // Histogram based on breadth of coverage
+ }
+ }
+
+ if (stats->tid != -1) {
+ set_read_counts(data, stats, n_bam_files);
+ if (opt_print_histogram) {
+ print_hist(file_out, h, stats, hist, n_bins, opt_full_utf);
+ } else if (opt_print_tabular) {
+ print_tabular_line(file_out, h, stats);
+ }
+ }
+
+
+ if (!opt_reg && opt_print_tabular) {
+ memset(stats, 0, sizeof(stats_aux_t));
+ for (i = 0; i < n_targets; ++i) {
+ if (!covered_tids[i]) {
+ stats->tid = i;
+ stats->end = sam_hdr_tid2len(h, i);
+ print_tabular_line(file_out, h, stats);
+ }
+ }
+ }
+
+ if (ret < 0) status = EXIT_FAILURE;
+
+coverage_end:
+ if (n_plp) free(n_plp);
+ if (plp) free(plp);
+ bam_mplp_destroy(mplp);
+
+ if (covered_tids) free(covered_tids);
+ if (hist) free(hist);
+ if (stats) free(stats);
+
+
+ // Close files and free data structures
+ if (!(file_out == samtools_stdout || fclose(file_out) == 0)) {
+ if (status == EXIT_SUCCESS) {
+ print_error_errno("coverage", "error on closing \"%s\"",
+ (opt_output_file && strcmp(opt_output_file, "-") != 0?
+ opt_output_file : "samtools_stdout"));
+ status = EXIT_FAILURE;
+ }
+ }
+
+ if (data) {
+ for (i = 0; i < n_bam_files && data[i]; ++i) {
+ sam_hdr_destroy(data[i]->hdr);
+ if (data[i]->fp) sam_close(data[i]->fp);
+ hts_itr_destroy(data[i]->iter);
+ free(data[i]);
+ }
+ free(data);
+ }
+
+ if (opt_file_list && fn) {
+ for (i = 0; i < n_bam_files; ++i)
+ free(fn[i]);
+ free(fn);
+ }
+ sam_global_args_free(&ga);
+
+ return status;
+}
+
+#ifdef _MAIN_BAMCOV
+int samtools_coverage_main(int argc, char *argv[]) {
+ return main_coverage(argc, argv);
+}
+#endif
/* cut_target.c -- targetcut subcommand.
Copyright (C) 2011 Broad Institute.
- Copyright (C) 2012-2013, 2015, 2016 Genome Research Ltd.
+ Copyright (C) 2012-2013, 2015, 2016, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
int min_baseQ, tid, max_bases;
uint16_t *bases;
samFile *fp;
- bam_hdr_t *h;
+ sam_hdr_t *h;
char *ref;
- int len;
+ hts_pos_t len;
faidx_t *fai;
errmod_t *em;
} ct_t;
return ret<<8|k;
}
-static void process_cns(bam_hdr_t *h, int tid, int l, uint16_t *cns)
+static void process_cns(sam_hdr_t *h, int tid, hts_pos_t l, uint16_t *cns)
{
- int i, f[2][2], *prev, *curr, *swap_tmp, s;
+ int64_t i, s;
+ int f[2][2], *prev, *curr, *swap_tmp;
uint8_t *b; // backtrack array
b = calloc(l, 1);
f[0][0] = f[0][1] = 0;
s = b[i]>>s&1;
}
// print
- for (i = 0, s = -1; i < INT_MAX && i <= l; ++i) {
+ for (i = 0, s = -1; i < INT64_MAX && i <= l; ++i) {
if (i == l || ((b[i]>>2&3) == 0 && s >= 0)) {
if (s >= 0) {
- int j;
- printf("%s:%d-%d\t0\t%s\t%d\t60\t%dM\t*\t0\t0\t", h->target_name[tid], s+1, i, h->target_name[tid], s+1, i-s);
+ int64_t j;
+ printf("%s:%"PRId64"-%"PRId64"\t0\t%s\t%"PRId64"\t60\t%"PRId64"M\t*\t0\t0\t", sam_hdr_tid2name(h, tid), s+1, i, sam_hdr_tid2name(h, tid), s+1, i-s);
for (j = s; j < i; ++j) {
int c = cns[j]>>8;
if (c == 0) putchar('N');
if ( g->fai && b->core.tid >= 0 ) {
if (b->core.tid != g->tid) { // then load the sequence
free(g->ref);
- g->ref = fai_fetch(g->fai, g->h->target_name[b->core.tid], &g->len);
+ g->ref = fai_fetch64(g->fai, sam_hdr_tid2name(g->h, b->core.tid), &g->len);
g->tid = b->core.tid;
}
sam_prob_realn(b, g->ref, g->len, 1<<1|1);
int main_cut_target(int argc, char *argv[])
{
- int c, tid, pos, n, lasttid = -1, l, max_l, usage = 0;
+ int c, tid, pos, n, lasttid = -1, usage = 0;
+ hts_pos_t l, max_l;
const bam_pileup1_t *p;
bam_plp_t plp;
uint16_t *cns;
}
if (usage || argc == optind) {
fprintf(stderr, "Usage: samtools targetcut [-Q minQ] [-i inPen] [-0 em0] [-1 em1] [-2 em2] <in.bam>\n");
- sam_global_opt_help(stderr, "-.--f-");
+ sam_global_opt_help(stderr, "-.--f--.");
return 1;
}
l = max_l = 0; cns = 0;
if (tid < 0) break;
if (tid != lasttid) { // change of chromosome
if (cns) process_cns(g.h, lasttid, l, cns);
- if (max_l < g.h->target_len[tid]) {
- max_l = g.h->target_len[tid];
+ if (max_l < sam_hdr_tid2len(g.h, tid)) {
+ max_l = sam_hdr_tid2len(g.h, tid);
kroundup32(max_l);
cns = realloc(cns, max_l * 2);
}
- l = g.h->target_len[tid];
+ l = sam_hdr_tid2len(g.h, tid);
memset(cns, 0, max_l * 2);
lasttid = tid;
}
}
process_cns(g.h, lasttid, l, cns);
free(cns);
- bam_hdr_destroy(g.h);
+ sam_hdr_destroy(g.h);
bam_plp_destroy(plp);
sam_close(g.fp);
if (g.fai) {
/* cut_target.c -- targetcut subcommand.
Copyright (C) 2011 Broad Institute.
- Copyright (C) 2012-2013, 2015, 2016 Genome Research Ltd.
+ Copyright (C) 2012-2013, 2015, 2016, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
int min_baseQ, tid, max_bases;
uint16_t *bases;
samFile *fp;
- bam_hdr_t *h;
+ sam_hdr_t *h;
char *ref;
- int len;
+ hts_pos_t len;
faidx_t *fai;
errmod_t *em;
} ct_t;
return ret<<8|k;
}
-static void process_cns(bam_hdr_t *h, int tid, int l, uint16_t *cns)
+static void process_cns(sam_hdr_t *h, int tid, hts_pos_t l, uint16_t *cns)
{
- int i, f[2][2], *prev, *curr, *swap_tmp, s;
+ int64_t i, s;
+ int f[2][2], *prev, *curr, *swap_tmp;
uint8_t *b; // backtrack array
b = calloc(l, 1);
f[0][0] = f[0][1] = 0;
s = b[i]>>s&1;
}
// print
- for (i = 0, s = -1; i < INT_MAX && i <= l; ++i) {
+ for (i = 0, s = -1; i < INT64_MAX && i <= l; ++i) {
if (i == l || ((b[i]>>2&3) == 0 && s >= 0)) {
if (s >= 0) {
- int j;
- fprintf(samtools_stdout, "%s:%d-%d\t0\t%s\t%d\t60\t%dM\t*\t0\t0\t", h->target_name[tid], s+1, i, h->target_name[tid], s+1, i-s);
+ int64_t j;
+ fprintf(samtools_stdout, "%s:%"PRId64"-%"PRId64"\t0\t%s\t%"PRId64"\t60\t%"PRId64"M\t*\t0\t0\t", sam_hdr_tid2name(h, tid), s+1, i, sam_hdr_tid2name(h, tid), s+1, i-s);
for (j = s; j < i; ++j) {
int c = cns[j]>>8;
if (c == 0) fputc('N', samtools_stdout);
if ( g->fai && b->core.tid >= 0 ) {
if (b->core.tid != g->tid) { // then load the sequence
free(g->ref);
- g->ref = fai_fetch(g->fai, g->h->target_name[b->core.tid], &g->len);
+ g->ref = fai_fetch64(g->fai, sam_hdr_tid2name(g->h, b->core.tid), &g->len);
g->tid = b->core.tid;
}
sam_prob_realn(b, g->ref, g->len, 1<<1|1);
int main_cut_target(int argc, char *argv[])
{
- int c, tid, pos, n, lasttid = -1, l, max_l, usage = 0;
+ int c, tid, pos, n, lasttid = -1, usage = 0;
+ hts_pos_t l, max_l;
const bam_pileup1_t *p;
bam_plp_t plp;
uint16_t *cns;
}
if (usage || argc == optind) {
fprintf(samtools_stderr, "Usage: samtools targetcut [-Q minQ] [-i inPen] [-0 em0] [-1 em1] [-2 em2] <in.bam>\n");
- sam_global_opt_help(samtools_stderr, "-.--f-");
+ sam_global_opt_help(samtools_stderr, "-.--f--.");
return 1;
}
l = max_l = 0; cns = 0;
if (tid < 0) break;
if (tid != lasttid) { // change of chromosome
if (cns) process_cns(g.h, lasttid, l, cns);
- if (max_l < g.h->target_len[tid]) {
- max_l = g.h->target_len[tid];
+ if (max_l < sam_hdr_tid2len(g.h, tid)) {
+ max_l = sam_hdr_tid2len(g.h, tid);
kroundup32(max_l);
cns = realloc(cns, max_l * 2);
}
- l = g.h->target_len[tid];
+ l = sam_hdr_tid2len(g.h, tid);
memset(cns, 0, max_l * 2);
lasttid = tid;
}
}
process_cns(g.h, lasttid, l, cns);
free(cns);
- bam_hdr_destroy(g.h);
+ sam_hdr_destroy(g.h);
bam_plp_destroy(plp);
sam_close(g.fp);
if (g.fai) {
hts_md5_destroy(md5);
if (args->output_fname) fclose(out);
+ gzclose(fp);
}
static int dict_usage(void)
hts_md5_destroy(md5);
if (args->output_fname) fclose(out);
+ gzclose(fp);
}
static int dict_usage(void)
/* faidx.c -- faidx subcommand.
- Copyright (C) 2008, 2009, 2013, 2016, 2018 Genome Research Ltd.
+ Copyright (C) 2008, 2009, 2013, 2016, 2018-2019 Genome Research Ltd.
Portions copyright (C) 2011 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
};
-static void reverse_complement(char *str, int len) {
+static void reverse_complement(char *str, const hts_pos_t len) {
char c;
- int i = 0, j = len - 1;
+ hts_pos_t i = 0, j = len - 1;
while (i <= j) {
c = str[i];
}
}
-
-static void reverse(char *str, int len) {
+static void reverse(char *str, const hts_pos_t len) {
char c;
- int i = 0, j = len - 1;
+ hts_pos_t i = 0, j = len - 1;
while (i < j) {
c = str[i];
}
-static int write_line(FILE *file, const char *line, const char *name, const int ignore,
- const int length, const int seq_len) {
- int beg, end;
+static int write_line(faidx_t *faid, FILE *file, const char *line, const char *name,
+ const int ignore, const int length, const hts_pos_t seq_len) {
+ int id;
+ hts_pos_t beg, end;
if (seq_len < 0) {
fprintf(stderr, "[faidx] Failed to fetch sequence in %s\n", name);
}
} else if (seq_len == 0) {
fprintf(stderr, "[faidx] Zero length sequence: %s\n", name);
- } else if (hts_parse_reg(name, &beg, &end) && (end < INT_MAX) && (seq_len != end - beg)) {
+ } else if (fai_parse_region(faid, name, &id, &beg, &end, 0)
+ && (end < INT_MAX) && (seq_len != end - beg)) {
fprintf(stderr, "[faidx] Truncated sequence: %s\n", name);
}
- size_t i, seq_sz = seq_len;
+ hts_pos_t i, seq_sz = seq_len;
for (i = 0; i < seq_sz; i += length)
{
- size_t len = i + length < seq_sz ? length : seq_sz - i;
+ hts_pos_t len = i + length < seq_sz ? length : seq_sz - i;
if (fwrite(line + i, 1, len, file) < len ||
fputc('\n', file) == EOF) {
print_error_errno("faidx", "failed to write output");
const int length, const int rev,
const char *pos_strand_name, const char *neg_strand_name,
enum fai_format_options format) {
- int seq_len;
- char *seq = fai_fetch(faid, name, &seq_len);
+ hts_pos_t seq_len;
+ char *seq = fai_fetch64(faid, name, &seq_len);
if (format == FAI_FASTA) {
fprintf(file, ">%s%s\n", name, rev ? neg_strand_name : pos_strand_name);
reverse_complement(seq, seq_len);
}
- if (write_line(file, seq, name, ignore, length, seq_len) == EXIT_FAILURE) {
+ if (write_line(faid, file, seq, name, ignore, length, seq_len)
+ == EXIT_FAILURE) {
free(seq);
return EXIT_FAILURE;
}
if (format == FAI_FASTQ) {
fprintf(file, "+\n");
- char *qual = fai_fetchqual(faid, name, &seq_len);
+ char *qual = fai_fetchqual64(faid, name, &seq_len);
if (rev && seq_len > 0) {
reverse(qual, seq_len);
}
- if (write_line(file, qual, name, ignore, length, seq_len) == EXIT_FAILURE) {
- free(seq);
+ if (write_line(faid, file, qual, name, ignore, length, seq_len)
+ == EXIT_FAILURE) {
+ free(qual);
return EXIT_FAILURE;
}
/* faidx.c -- faidx subcommand.
- Copyright (C) 2008, 2009, 2013, 2016, 2018 Genome Research Ltd.
+ Copyright (C) 2008, 2009, 2013, 2016, 2018-2019 Genome Research Ltd.
Portions copyright (C) 2011 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
};
-static void reverse_complement(char *str, int len) {
+static void reverse_complement(char *str, const hts_pos_t len) {
char c;
- int i = 0, j = len - 1;
+ hts_pos_t i = 0, j = len - 1;
while (i <= j) {
c = str[i];
}
}
-
-static void reverse(char *str, int len) {
+static void reverse(char *str, const hts_pos_t len) {
char c;
- int i = 0, j = len - 1;
+ hts_pos_t i = 0, j = len - 1;
while (i < j) {
c = str[i];
}
-static int write_line(FILE *file, const char *line, const char *name, const int ignore,
- const int length, const int seq_len) {
- int beg, end;
+static int write_line(faidx_t *faid, FILE *file, const char *line, const char *name,
+ const int ignore, const int length, const hts_pos_t seq_len) {
+ int id;
+ hts_pos_t beg, end;
if (seq_len < 0) {
fprintf(samtools_stderr, "[faidx] Failed to fetch sequence in %s\n", name);
}
} else if (seq_len == 0) {
fprintf(samtools_stderr, "[faidx] Zero length sequence: %s\n", name);
- } else if (hts_parse_reg(name, &beg, &end) && (end < INT_MAX) && (seq_len != end - beg)) {
+ } else if (fai_parse_region(faid, name, &id, &beg, &end, 0)
+ && (end < INT_MAX) && (seq_len != end - beg)) {
fprintf(samtools_stderr, "[faidx] Truncated sequence: %s\n", name);
}
- size_t i, seq_sz = seq_len;
+ hts_pos_t i, seq_sz = seq_len;
for (i = 0; i < seq_sz; i += length)
{
- size_t len = i + length < seq_sz ? length : seq_sz - i;
+ hts_pos_t len = i + length < seq_sz ? length : seq_sz - i;
if (fwrite(line + i, 1, len, file) < len ||
fputc('\n', file) == EOF) {
print_error_errno("faidx", "failed to write output");
const int length, const int rev,
const char *pos_strand_name, const char *neg_strand_name,
enum fai_format_options format) {
- int seq_len;
- char *seq = fai_fetch(faid, name, &seq_len);
+ hts_pos_t seq_len;
+ char *seq = fai_fetch64(faid, name, &seq_len);
if (format == FAI_FASTA) {
fprintf(file, ">%s%s\n", name, rev ? neg_strand_name : pos_strand_name);
reverse_complement(seq, seq_len);
}
- if (write_line(file, seq, name, ignore, length, seq_len) == EXIT_FAILURE) {
+ if (write_line(faid, file, seq, name, ignore, length, seq_len)
+ == EXIT_FAILURE) {
free(seq);
return EXIT_FAILURE;
}
if (format == FAI_FASTQ) {
fprintf(file, "+\n");
- char *qual = fai_fetchqual(faid, name, &seq_len);
+ char *qual = fai_fetchqual64(faid, name, &seq_len);
if (rev && seq_len > 0) {
reverse(qual, seq_len);
}
- if (write_line(file, qual, name, ignore, length, seq_len) == EXIT_FAILURE) {
- free(seq);
+ if (write_line(faid, file, qual, name, ignore, length, seq_len)
+ == EXIT_FAILURE) {
+ free(qual);
return EXIT_FAILURE;
}
--- /dev/null
+[Files in this distribution outwith the cram/ subdirectory are distributed
+according to the terms of the following MIT/Expat license.]
+
+The MIT/Expat License
+
+Copyright (C) 2012-2019 Genome Research Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+
+[Files within the cram/ subdirectory in this distribution are distributed
+according to the terms of the following Modified 3-Clause BSD license.]
+
+The Modified-BSD License
+
+Copyright (C) 2012-2019 Genome Research Ltd.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+3. Neither the names Genome Research Ltd and Wellcome Trust Sanger Institute
+ nor the names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR ITS CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+[The use of a range of years within a copyright notice in this distribution
+should be interpreted as being equivalent to a list of years including the
+first and last year specified and all consecutive years between them.
+
+For example, a copyright notice that reads "Copyright (C) 2005, 2007-2009,
+2011-2012" should be interpreted as being identical to a notice that reads
+"Copyright (C) 2005, 2007, 2008, 2009, 2011, 2012" and a copyright notice
+that reads "Copyright (C) 2005-2012" should be interpreted as being identical
+to a notice that reads "Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012".]
--- /dev/null
+HTSlib is an implementation of a unified C library for accessing common file
+formats, such as SAM, CRAM, VCF, and BCF, used for high-throughput sequencing
+data. It is the core library used by samtools and bcftools.
+
+See INSTALL for building and installation instructions.
+++ /dev/null
-[Files in this distribution outwith the cram/ subdirectory are distributed
-according to the terms of the following MIT/Expat license.]
-
-The MIT/Expat License
-
-Copyright (C) 2012-2018 Genome Research Ltd.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
-
-[Files within the cram/ subdirectory in this distribution are distributed
-according to the terms of the following Modified 3-Clause BSD license.]
-
-The Modified-BSD License
-
-Copyright (C) 2012-2018 Genome Research Ltd.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-3. Neither the names Genome Research Ltd and Wellcome Trust Sanger Institute
- nor the names of its contributors may be used to endorse or promote products
- derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR ITS CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-[The use of a range of years within a copyright notice in this distribution
-should be interpreted as being equivalent to a list of years including the
-first and last year specified and all consecutive years between them.
-
-For example, a copyright notice that reads "Copyright (C) 2005, 2007-2009,
-2011-2012" should be interpreted as being identical to a notice that reads
-"Copyright (C) 2005, 2007, 2008, 2009, 2011, 2012" and a copyright notice
-that reads "Copyright (C) 2005-2012" should be interpreted as being identical
-to a notice that reads "Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012".]
+++ /dev/null
-HTSlib is an implementation of a unified C library for accessing common file
-formats, such as SAM, CRAM, VCF, and BCF, used for high-throughput sequencing
-data. It is the core library used by samtools and bcftools.
-
-See INSTALL for building and installation instructions.
+++ /dev/null
-/* The MIT License
-
- Copyright (c) 2011 Heng Li <lh3@live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-#include <config.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <zlib.h>
-#include "htslib/kstring.h"
-#include "htslib/kseq.h"
-KSTREAM_INIT(gzFile, gzread, 16384)
-
-#define N_TMPSTR 5
-#define LINE_LEN 60
-
-// append a CIGAR operation plus length
-#define write_cigar(_c, _n, _m, _v) do { \
- if (_n == _m) { \
- _m = _m? _m<<1 : 4; \
- _c = realloc(_c, _m * sizeof(unsigned)); \
- } \
- _c[_n++] = (_v); \
- } while (0)
-
-// a fatal error
-static void fatal(const char *msg)
-{
- fprintf(stderr, "E %s\n", msg);
- exit(1);
-}
-// remove pads
-static void remove_pads(const kstring_t *src, kstring_t *dst)
-{
- int i, j;
- dst->l = 0;
- kputsn(src->s, src->l, dst);
- for (i = j = 0; i < dst->l; ++i)
- if (dst->s[i] != '*') dst->s[j++] = dst->s[i];
- dst->s[j] = 0;
- dst->l = j;
-}
-
-int main(int argc, char *argv[])
-{
- gzFile fp;
- kstream_t *ks;
- kstring_t s, t[N_TMPSTR];
- int dret, i, k, af_n, af_max, af_i, c, is_padded = 0, write_cns = 0, *p2u = 0;
- long m_cigar = 0, n_cigar = 0;
- unsigned *af, *cigar = 0;
-
- while ((c = getopt(argc, argv, "pc")) >= 0) {
- switch (c) {
- case 'p': is_padded = 1; break;
- case 'c': write_cns = 1; break;
- }
- }
- if (argc == optind) {
- fprintf(stderr, "\nUsage: ace2sam [-pc] <in.ace>\n\n");
- fprintf(stderr, "Options: -p output padded SAM\n");
- fprintf(stderr, " -c write the contig sequence in SAM\n\n");
- fprintf(stderr, "Notes: 1. Fields must appear in the following order: (CO->[BQ]->(AF)->(RD->QA))\n");
- fprintf(stderr, " 2. The order of reads in AF and in RD must be identical\n");
- fprintf(stderr, " 3. Except in BQ, words and numbers must be separated by a single SPACE or TAB\n");
- fprintf(stderr, " 4. This program writes the headerless SAM to stdout and header to stderr\n\n");
- return 1;
- }
-
- s.l = s.m = 0; s.s = 0;
- af_n = af_max = af_i = 0; af = 0;
- for (i = 0; i < N_TMPSTR; ++i) t[i].l = t[i].m = 0, t[i].s = 0;
- fp = strcmp(argv[1], "-")? gzopen(argv[optind], "r") : gzdopen(fileno(stdin), "r");
- ks = ks_init(fp);
- while (ks_getuntil(ks, 0, &s, &dret) >= 0) {
- if (strcmp(s.s, "CO") == 0) { // contig sequence
- kstring_t *cns;
- t[0].l = t[1].l = t[2].l = t[3].l = t[4].l = 0; // 0: name; 1: padded ctg; 2: unpadded ctg/padded read; 3: unpadded read; 4: SAM line
- af_n = af_i = 0; // reset the af array
- ks_getuntil(ks, 0, &s, &dret); kputs(s.s, &t[0]); // contig name
- ks_getuntil(ks, '\n', &s, &dret); // read the whole line
- while (ks_getuntil(ks, '\n', &s, &dret) >= 0 && s.l > 0) kputsn(s.s, s.l, &t[1]); // read the padded consensus sequence
- remove_pads(&t[1], &t[2]); // construct the unpadded sequence
- // compute the array for mapping padded positions to unpadded positions
- p2u = realloc(p2u, t[1].m * sizeof(int));
- for (i = k = 0; i < t[1].l; ++i) {
- p2u[i] = k;
- if (t[1].s[i] != '*') ++k;
- }
- // write out the SAM header and contig sequences
- fprintf(stderr, "H @SQ\tSN:%s\tLN:%llu\n", t[0].s, (unsigned long long)(t[is_padded?1:2].l)); // The SAM header line
- cns = &t[is_padded?1:2];
- fprintf(stderr, "S >%s\n", t[0].s);
- for (i = 0; i < cns->l; i += LINE_LEN) {
- fputs("S ", stderr);
- for (k = 0; k < LINE_LEN && i + k < cns->l; ++k)
- fputc(cns->s[i + k], stderr);
- fputc('\n', stderr);
- }
-
-#define __padded2cigar(sp) do { \
- int i, l_M = 0, l_D = 0; \
- for (i = 0; i < sp.l; ++i) { \
- if (sp.s[i] == '*') { \
- if (l_M) write_cigar(cigar, n_cigar, m_cigar, l_M<<4); \
- ++l_D; l_M = 0; \
- } else { \
- if (l_D) write_cigar(cigar, n_cigar, m_cigar, l_D<<4 | 2); \
- ++l_M; l_D = 0; \
- } \
- } \
- if (l_M) write_cigar(cigar, n_cigar, m_cigar, l_M<<4); \
- else write_cigar(cigar, n_cigar, m_cigar, l_D<<4 | 2); \
- } while (0)
-
- if (write_cns) { // write the consensus SAM line (dummy read)
- n_cigar = 0;
- if (is_padded) __padded2cigar(t[1]);
- else write_cigar(cigar, n_cigar, m_cigar, t[2].l<<4);
- kputsn(t[0].s, t[0].l, &t[4]); kputs("\t516\t", &t[4]); kputsn(t[0].s, t[0].l, &t[4]); kputs("\t1\t60\t", &t[4]);
- for (i = 0; i < n_cigar; ++i) {
- kputw(cigar[i]>>4, &t[4]); kputc("MIDNSHP=X"[cigar[i]&0xf], &t[4]);
- }
- kputs("\t*\t0\t0\t", &t[4]); kputsn(t[2].s, t[2].l, &t[4]); kputs("\t*", &t[4]);
- }
- } else if (strcmp(s.s, "BQ") == 0) { // contig quality
- if (t[0].l == 0) fatal("come to 'BQ' before reading 'CO'");
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret); // read the entire "BQ" line
- if (write_cns) t[4].s[--t[4].l] = 0; // remove the trailing "*"
- for (i = 0; i < t[2].l; ++i) { // read the consensus quality
- int q;
- if (ks_getuntil(ks, 0, &s, &dret) < 0) fprintf(stderr, "E truncated contig quality\n");
- if (s.l) {
- q = atoi(s.s) + 33;
- if (q > 126) q = 126;
- if (write_cns) kputc(q, &t[4]);
- } else --i;
- }
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- ks_getuntil(ks, '\n', &s, &dret); // skip the empty line
- if (write_cns) {
- if (t[4].l) puts(t[4].s);
- t[4].l = 0;
- }
- } else if (strcmp(s.s, "AF") == 0) { // padded read position
- int reversed, neg, pos;
- if (t[0].l == 0) fatal("come to 'AF' before reading 'CO'");
- if (write_cns) {
- if (t[4].l) puts(t[4].s);
- t[4].l = 0;
- }
- ks_getuntil(ks, 0, &s, &dret); // read name
- ks_getuntil(ks, 0, &s, &dret); reversed = s.s[0] == 'C'? 1 : 0; // strand
- ks_getuntil(ks, 0, &s, &dret); pos = atoi(s.s); neg = pos < 0? 1 : 0; pos = pos < 0? -pos : pos; // position
- if (af_n == af_max) { // double the af array
- af_max = af_max? af_max<<1 : 4;
- af = realloc(af, af_max * sizeof(unsigned));
- }
- af[af_n++] = pos << 2 | neg << 1 | reversed; // keep the placement information
- } else if (strcmp(s.s, "RD") == 0) { // read sequence
- if (af_i >= af_n) fatal("more 'RD' records than 'AF'");
- t[2].l = t[3].l = t[4].l = 0;
- ks_getuntil(ks, 0, &t[4], &dret); // QNAME
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret); // read the entire RD line
- while (ks_getuntil(ks, '\n', &s, &dret) >= 0 && s.l > 0) kputs(s.s, &t[2]); // read the read sequence
- } else if (strcmp(s.s, "QA") == 0) { // clipping
- if (af_i >= af_n) fatal("more 'QA' records than 'AF'");
- int beg, end, pos, op;
- ks_getuntil(ks, 0, &s, &dret); ks_getuntil(ks, 0, &s, &dret); // skip quality clipping
- ks_getuntil(ks, 0, &s, &dret); beg = atoi(s.s) - 1; // align clipping start
- ks_getuntil(ks, 0, &s, &dret); end = atoi(s.s); // clipping end
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- // compute 1-based POS
- pos = af[af_i]>>2; // retrieve the position information
- if (af[af_i]>>1&1) pos = -pos;
- pos += beg; // now pos is the true padded position
- // generate CIGAR
- remove_pads(&t[2], &t[3]); // backup the unpadded read sequence
- n_cigar = 0;
- if (beg) write_cigar(cigar, n_cigar, m_cigar, beg<<4|4);
- if (is_padded) {
- __padded2cigar(t[2]);
- if (beg && n_cigar > 1) cigar[1] -= beg<<4; // fix the left-hand CIGAR
- if (end < t[2].l && n_cigar) cigar[n_cigar-1] -= (t[2].l - end)<<4; // fix the right-hand CIGAR
- } else {
- // generate flattened CIGAR string
- for (i = beg, k = pos - 1; i < end; ++i, ++k)
- t[2].s[i] = t[2].s[i] != '*'? (t[1].s[k] != '*'? 0 : 1) : (t[1].s[k] != '*'? 2 : 6);
- // generate the proper CIGAR
- for (i = beg + 1, k = 1, op = t[2].s[beg]; i < end; ++i) {
- if (op != t[2].s[i]) {
- write_cigar(cigar, n_cigar, m_cigar, k<<4|op);
- op = t[2].s[i]; k = 1;
- } else ++k;
- }
- write_cigar(cigar, n_cigar, m_cigar, k<<4|op);
- // remove unnecessary "P" and possibly merge adjacent operations
- for (i = 2; i < n_cigar; ++i) {
- if ((cigar[i]&0xf) != 1 && (cigar[i-1]&0xf) == 6 && (cigar[i-2]&0xf) != 1) {
- cigar[i-1] = 0;
- if ((cigar[i]&0xf) == (cigar[i-2]&0xf)) // merge operations
- cigar[i] += cigar[i-2], cigar[i-2] = 0;
- }
- }
- for (i = k = 0; i < n_cigar; ++i) // squeeze out dumb operations
- if (cigar[i]) cigar[k++] = cigar[i];
- n_cigar = k;
- }
- if (end < t[2].l) write_cigar(cigar, n_cigar, m_cigar, (t[2].l - end)<<4|4);
- // write the SAM line for the read
- kputc('\t', &t[4]); // QNAME has already been written
- kputw((af[af_i]&1)? 16 : 0, &t[4]); kputc('\t', &t[4]); // FLAG
- kputsn(t[0].s, t[0].l, &t[4]); kputc('\t', &t[4]); // RNAME
- kputw(is_padded? pos : p2u[pos-1]+1, &t[4]); // POS
- kputs("\t60\t", &t[4]); // MAPQ
- for (i = 0; i < n_cigar; ++i) { // CIGAR
- kputw(cigar[i]>>4, &t[4]); kputc("MIDNSHP=X"[cigar[i]&0xf], &t[4]);
- }
- kputs("\t*\t0\t0\t", &t[4]); // empty MRNM, MPOS and TLEN
- kputsn(t[3].s, t[3].l, &t[4]); // unpadded SEQ
- kputs("\t*", &t[4]); // QUAL
- puts(t[4].s); // print to stdout
- ++af_i;
- } else if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- }
- ks_destroy(ks);
- gzclose(fp);
- free(af); free(s.s); free(cigar); free(p2u);
- for (i = 0; i < N_TMPSTR; ++i) free(t[i].s);
- return 0;
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* The MIT License
-
- Copyright (c) 2011 Heng Li <lh3@live.co.uk>
-
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
-*/
-
-#include <config.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <zlib.h>
-#include "htslib/kstring.h"
-#include "htslib/kseq.h"
-KSTREAM_INIT(gzFile, gzread, 16384)
-
-#define N_TMPSTR 5
-#define LINE_LEN 60
-
-// append a CIGAR operation plus length
-#define write_cigar(_c, _n, _m, _v) do { \
- if (_n == _m) { \
- _m = _m? _m<<1 : 4; \
- _c = realloc(_c, _m * sizeof(unsigned)); \
- } \
- _c[_n++] = (_v); \
- } while (0)
-
-// a fatal error
-static void fatal(const char *msg)
-{
- fprintf(samtools_stderr, "E %s\n", msg);
- exit(1);
-}
-// remove pads
-static void remove_pads(const kstring_t *src, kstring_t *dst)
-{
- int i, j;
- dst->l = 0;
- kputsn(src->s, src->l, dst);
- for (i = j = 0; i < dst->l; ++i)
- if (dst->s[i] != '*') dst->s[j++] = dst->s[i];
- dst->s[j] = 0;
- dst->l = j;
-}
-
-int samtools_ace2sam_main(int argc, char *argv[])
-{
- gzFile fp;
- kstream_t *ks;
- kstring_t s, t[N_TMPSTR];
- int dret, i, k, af_n, af_max, af_i, c, is_padded = 0, write_cns = 0, *p2u = 0;
- long m_cigar = 0, n_cigar = 0;
- unsigned *af, *cigar = 0;
-
- while ((c = getopt(argc, argv, "pc")) >= 0) {
- switch (c) {
- case 'p': is_padded = 1; break;
- case 'c': write_cns = 1; break;
- }
- }
- if (argc == optind) {
- fprintf(samtools_stderr, "\nUsage: ace2sam [-pc] <in.ace>\n\n");
- fprintf(samtools_stderr, "Options: -p output padded SAM\n");
- fprintf(samtools_stderr, " -c write the contig sequence in SAM\n\n");
- fprintf(samtools_stderr, "Notes: 1. Fields must appear in the following order: (CO->[BQ]->(AF)->(RD->QA))\n");
- fprintf(samtools_stderr, " 2. The order of reads in AF and in RD must be identical\n");
- fprintf(samtools_stderr, " 3. Except in BQ, words and numbers must be separated by a single SPACE or TAB\n");
- fprintf(samtools_stderr, " 4. This program writes the headerless SAM to samtools_stdout and header to samtools_stderr\n\n");
- return 1;
- }
-
- s.l = s.m = 0; s.s = 0;
- af_n = af_max = af_i = 0; af = 0;
- for (i = 0; i < N_TMPSTR; ++i) t[i].l = t[i].m = 0, t[i].s = 0;
- fp = strcmp(argv[1], "-")? gzopen(argv[optind], "r") : gzdopen(fileno(stdin), "r");
- ks = ks_init(fp);
- while (ks_getuntil(ks, 0, &s, &dret) >= 0) {
- if (strcmp(s.s, "CO") == 0) { // contig sequence
- kstring_t *cns;
- t[0].l = t[1].l = t[2].l = t[3].l = t[4].l = 0; // 0: name; 1: padded ctg; 2: unpadded ctg/padded read; 3: unpadded read; 4: SAM line
- af_n = af_i = 0; // reset the af array
- ks_getuntil(ks, 0, &s, &dret); kputs(s.s, &t[0]); // contig name
- ks_getuntil(ks, '\n', &s, &dret); // read the whole line
- while (ks_getuntil(ks, '\n', &s, &dret) >= 0 && s.l > 0) kputsn(s.s, s.l, &t[1]); // read the padded consensus sequence
- remove_pads(&t[1], &t[2]); // construct the unpadded sequence
- // compute the array for mapping padded positions to unpadded positions
- p2u = realloc(p2u, t[1].m * sizeof(int));
- for (i = k = 0; i < t[1].l; ++i) {
- p2u[i] = k;
- if (t[1].s[i] != '*') ++k;
- }
- // write out the SAM header and contig sequences
- fprintf(samtools_stderr, "H @SQ\tSN:%s\tLN:%llu\n", t[0].s, (unsigned long long)(t[is_padded?1:2].l)); // The SAM header line
- cns = &t[is_padded?1:2];
- fprintf(samtools_stderr, "S >%s\n", t[0].s);
- for (i = 0; i < cns->l; i += LINE_LEN) {
- fputs("S ", samtools_stderr);
- for (k = 0; k < LINE_LEN && i + k < cns->l; ++k)
- fputc(cns->s[i + k], samtools_stderr);
- fputc('\n', samtools_stderr);
- }
-
-#define __padded2cigar(sp) do { \
- int i, l_M = 0, l_D = 0; \
- for (i = 0; i < sp.l; ++i) { \
- if (sp.s[i] == '*') { \
- if (l_M) write_cigar(cigar, n_cigar, m_cigar, l_M<<4); \
- ++l_D; l_M = 0; \
- } else { \
- if (l_D) write_cigar(cigar, n_cigar, m_cigar, l_D<<4 | 2); \
- ++l_M; l_D = 0; \
- } \
- } \
- if (l_M) write_cigar(cigar, n_cigar, m_cigar, l_M<<4); \
- else write_cigar(cigar, n_cigar, m_cigar, l_D<<4 | 2); \
- } while (0)
-
- if (write_cns) { // write the consensus SAM line (dummy read)
- n_cigar = 0;
- if (is_padded) __padded2cigar(t[1]);
- else write_cigar(cigar, n_cigar, m_cigar, t[2].l<<4);
- kputsn(t[0].s, t[0].l, &t[4]); kputs("\t516\t", &t[4]); kputsn(t[0].s, t[0].l, &t[4]); kputs("\t1\t60\t", &t[4]);
- for (i = 0; i < n_cigar; ++i) {
- kputw(cigar[i]>>4, &t[4]); kputc("MIDNSHP=X"[cigar[i]&0xf], &t[4]);
- }
- kputs("\t*\t0\t0\t", &t[4]); kputsn(t[2].s, t[2].l, &t[4]); kputs("\t*", &t[4]);
- }
- } else if (strcmp(s.s, "BQ") == 0) { // contig quality
- if (t[0].l == 0) fatal("come to 'BQ' before reading 'CO'");
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret); // read the entire "BQ" line
- if (write_cns) t[4].s[--t[4].l] = 0; // remove the trailing "*"
- for (i = 0; i < t[2].l; ++i) { // read the consensus quality
- int q;
- if (ks_getuntil(ks, 0, &s, &dret) < 0) fprintf(samtools_stderr, "E truncated contig quality\n");
- if (s.l) {
- q = atoi(s.s) + 33;
- if (q > 126) q = 126;
- if (write_cns) kputc(q, &t[4]);
- } else --i;
- }
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- ks_getuntil(ks, '\n', &s, &dret); // skip the empty line
- if (write_cns) {
- if (t[4].l) samtools_puts(t[4].s);
- t[4].l = 0;
- }
- } else if (strcmp(s.s, "AF") == 0) { // padded read position
- int reversed, neg, pos;
- if (t[0].l == 0) fatal("come to 'AF' before reading 'CO'");
- if (write_cns) {
- if (t[4].l) samtools_puts(t[4].s);
- t[4].l = 0;
- }
- ks_getuntil(ks, 0, &s, &dret); // read name
- ks_getuntil(ks, 0, &s, &dret); reversed = s.s[0] == 'C'? 1 : 0; // strand
- ks_getuntil(ks, 0, &s, &dret); pos = atoi(s.s); neg = pos < 0? 1 : 0; pos = pos < 0? -pos : pos; // position
- if (af_n == af_max) { // double the af array
- af_max = af_max? af_max<<1 : 4;
- af = realloc(af, af_max * sizeof(unsigned));
- }
- af[af_n++] = pos << 2 | neg << 1 | reversed; // keep the placement information
- } else if (strcmp(s.s, "RD") == 0) { // read sequence
- if (af_i >= af_n) fatal("more 'RD' records than 'AF'");
- t[2].l = t[3].l = t[4].l = 0;
- ks_getuntil(ks, 0, &t[4], &dret); // QNAME
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret); // read the entire RD line
- while (ks_getuntil(ks, '\n', &s, &dret) >= 0 && s.l > 0) kputs(s.s, &t[2]); // read the read sequence
- } else if (strcmp(s.s, "QA") == 0) { // clipping
- if (af_i >= af_n) fatal("more 'QA' records than 'AF'");
- int beg, end, pos, op;
- ks_getuntil(ks, 0, &s, &dret); ks_getuntil(ks, 0, &s, &dret); // skip quality clipping
- ks_getuntil(ks, 0, &s, &dret); beg = atoi(s.s) - 1; // align clipping start
- ks_getuntil(ks, 0, &s, &dret); end = atoi(s.s); // clipping end
- if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- // compute 1-based POS
- pos = af[af_i]>>2; // retrieve the position information
- if (af[af_i]>>1&1) pos = -pos;
- pos += beg; // now pos is the true padded position
- // generate CIGAR
- remove_pads(&t[2], &t[3]); // backup the unpadded read sequence
- n_cigar = 0;
- if (beg) write_cigar(cigar, n_cigar, m_cigar, beg<<4|4);
- if (is_padded) {
- __padded2cigar(t[2]);
- if (beg && n_cigar > 1) cigar[1] -= beg<<4; // fix the left-hand CIGAR
- if (end < t[2].l && n_cigar) cigar[n_cigar-1] -= (t[2].l - end)<<4; // fix the right-hand CIGAR
- } else {
- // generate flattened CIGAR string
- for (i = beg, k = pos - 1; i < end; ++i, ++k)
- t[2].s[i] = t[2].s[i] != '*'? (t[1].s[k] != '*'? 0 : 1) : (t[1].s[k] != '*'? 2 : 6);
- // generate the proper CIGAR
- for (i = beg + 1, k = 1, op = t[2].s[beg]; i < end; ++i) {
- if (op != t[2].s[i]) {
- write_cigar(cigar, n_cigar, m_cigar, k<<4|op);
- op = t[2].s[i]; k = 1;
- } else ++k;
- }
- write_cigar(cigar, n_cigar, m_cigar, k<<4|op);
- // remove unnecessary "P" and possibly merge adjacent operations
- for (i = 2; i < n_cigar; ++i) {
- if ((cigar[i]&0xf) != 1 && (cigar[i-1]&0xf) == 6 && (cigar[i-2]&0xf) != 1) {
- cigar[i-1] = 0;
- if ((cigar[i]&0xf) == (cigar[i-2]&0xf)) // merge operations
- cigar[i] += cigar[i-2], cigar[i-2] = 0;
- }
- }
- for (i = k = 0; i < n_cigar; ++i) // squeeze out dumb operations
- if (cigar[i]) cigar[k++] = cigar[i];
- n_cigar = k;
- }
- if (end < t[2].l) write_cigar(cigar, n_cigar, m_cigar, (t[2].l - end)<<4|4);
- // write the SAM line for the read
- kputc('\t', &t[4]); // QNAME has already been written
- kputw((af[af_i]&1)? 16 : 0, &t[4]); kputc('\t', &t[4]); // FLAG
- kputsn(t[0].s, t[0].l, &t[4]); kputc('\t', &t[4]); // RNAME
- kputw(is_padded? pos : p2u[pos-1]+1, &t[4]); // POS
- kputs("\t60\t", &t[4]); // MAPQ
- for (i = 0; i < n_cigar; ++i) { // CIGAR
- kputw(cigar[i]>>4, &t[4]); kputc("MIDNSHP=X"[cigar[i]&0xf], &t[4]);
- }
- kputs("\t*\t0\t0\t", &t[4]); // empty MRNM, MPOS and TLEN
- kputsn(t[3].s, t[3].l, &t[4]); // unpadded SEQ
- kputs("\t*", &t[4]); // QUAL
- samtools_puts(t[4].s); // print to samtools_stdout
- ++af_i;
- } else if (dret != '\n') ks_getuntil(ks, '\n', &s, &dret);
- }
- ks_destroy(ks);
- gzclose(fp);
- free(af); free(s.s); free(cigar); free(p2u);
- for (i = 0; i < N_TMPSTR; ++i) free(t[i].s);
- return 0;
-}
/* padding.c -- depad subcommand.
Copyright (C) 2011, 2012 Broad Institute.
- Copyright (C) 2014-2016 Genome Research Ltd.
+ Copyright (C) 2014-2016, 2019 Genome Research Ltd.
Portions copyright (C) 2012, 2013 Peter Cock, The James Hutton Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include <string.h>
#include <assert.h>
#include <unistd.h>
+#include <inttypes.h>
#include <htslib/kstring.h>
#include <htslib/sam.h>
#include <htslib/faidx.h>
-#include "sam_header.h"
#include "sam_opts.h"
#include "samtools.h"
if (_n == _m) { \
_m = _m? _m<<1 : 4; \
_c = (uint32_t*)realloc(_c, _m * 4); \
+ if (!(_c)) { \
+ fprintf(stderr, "[depad] ERROR: Memory allocation failure.\n"); \
+ return -1; \
+ } \
} \
_c[_n++] = (_v); \
} while (0)
return length != s->l;
}
-int load_unpadded_ref(faidx_t *fai, char *ref_name, int ref_len, kstring_t *seq)
+int load_unpadded_ref(faidx_t *fai, const char *ref_name, hts_pos_t ref_len, kstring_t *seq)
{
char base;
char *fai_ref = 0;
- int fai_ref_len = 0, k;
+ hts_pos_t fai_ref_len = 0, k;
- fai_ref = fai_fetch(fai, ref_name, &fai_ref_len);
+ fai_ref = fai_fetch64(fai, ref_name, &fai_ref_len);
if (fai_ref_len != ref_len) {
- fprintf(stderr, "[depad] ERROR: FASTA sequence %s length %i, expected %i\n", ref_name, fai_ref_len, ref_len);
+ fprintf(stderr, "[depad] ERROR: FASTA sequence %s length %"PRIhts_pos", expected %"PRIhts_pos"\n", ref_name, fai_ref_len, ref_len);
free(fai_ref);
return -1;
}
return 0;
}
-int get_unpadded_len(faidx_t *fai, char *ref_name, int padded_len)
+hts_pos_t get_unpadded_len(faidx_t *fai, const char *ref_name, hts_pos_t padded_len)
{
char base;
char *fai_ref = 0;
- int fai_ref_len = 0, k;
- int bases=0, gaps=0;
+ hts_pos_t fai_ref_len = 0, k;
+ hts_pos_t bases=0, gaps=0;
- fai_ref = fai_fetch(fai, ref_name, &fai_ref_len);
+ fai_ref = fai_fetch64(fai, ref_name, &fai_ref_len);
if (fai_ref_len != padded_len) {
- fprintf(stderr, "[depad] ERROR: FASTA sequence '%s' length %i, expected %i\n", ref_name, fai_ref_len, padded_len);
+ fprintf(stderr, "[depad] ERROR: FASTA sequence '%s' length %"PRIhts_pos", expected %"PRIhts_pos"\n", ref_name, fai_ref_len, padded_len);
free(fai_ref);
return -1;
}
return posmap;
}
-int bam_pad2unpad(samFile *in, samFile *out, bam_hdr_t *h, faidx_t *fai)
+int bam_pad2unpad(samFile *in, samFile *out, sam_hdr_t *h, faidx_t *fai)
{
bam1_t *b = 0;
kstring_t r, q;
uint32_t *cigar = bam_get_cigar(b);
n2 = 0;
- if (b->core.pos == 0 && b->core.tid >= 0 && strcmp(bam_get_qname(b), h->target_name[b->core.tid]) == 0) {
+ if (b->core.pos == 0 && b->core.tid >= 0 && strcmp(bam_get_qname(b), sam_hdr_tid2name(h, b->core.tid)) == 0) {
// fprintf(stderr, "[depad] Found embedded reference '%s'\n", bam_get_qname(b));
r_tid = b->core.tid;
if (0!=unpad_seq(b, &r)) {
fprintf(stderr, "[depad] ERROR: Problem parsing SEQ and/or CIGAR in reference %s\n", bam_get_qname(b));
return -1;
};
- if (h->target_len[r_tid] != r.l) {
- fprintf(stderr, "[depad] ERROR: (Padded) length of '%s' is %u in BAM header, but %llu in embedded reference\n", bam_get_qname(b), h->target_len[r_tid], (unsigned long long)(r.l));
+ if (sam_hdr_tid2len(h, r_tid) != r.l) {
+ fprintf(stderr, "[depad] ERROR: (Padded) length of '%s' is %"PRId64" in BAM header, but %zu in embedded reference\n", bam_get_qname(b), (int64_t) sam_hdr_tid2len(h, r_tid), r.l);
return -1;
}
if (fai) {
// Check the embedded reference matches the FASTA file
- if (load_unpadded_ref(fai, h->target_name[b->core.tid], h->target_len[b->core.tid], &q)) {
- fprintf(stderr, "[depad] ERROR: Failed to load embedded reference '%s' from FASTA\n", h->target_name[b->core.tid]);
+ if (load_unpadded_ref(fai, sam_hdr_tid2name(h, b->core.tid), sam_hdr_tid2len(h, b->core.tid), &q)) {
+ fprintf(stderr, "[depad] ERROR: Failed to load embedded reference '%s' from FASTA\n", sam_hdr_tid2name(h, b->core.tid));
return -1;
}
assert(r.l == q.l);
if (r.s[i] != q.s[i]) {
// Show gaps as ASCII 45
fprintf(stderr, "[depad] ERROR: Embedded sequence and reference FASTA don't match for %s base %i, '%c' vs '%c'\n",
- h->target_name[b->core.tid], i+1,
+ sam_hdr_tid2name(h, b->core.tid), i+1,
r.s[i] ? seq_nt16_str[(int)r.s[i]] : 45,
q.s[i] ? seq_nt16_str[(int)q.s[i]] : 45);
return -1;
; // good case, reference available
//fprintf(stderr, "[depad] Have ref '%s' for read '%s'\n", h->target_name[b->core.tid], bam_get_qname(b));
} else if (fai) {
- if (load_unpadded_ref(fai, h->target_name[b->core.tid], h->target_len[b->core.tid], &r)) {
- fprintf(stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", h->target_name[b->core.tid]);
+ if (load_unpadded_ref(fai, sam_hdr_tid2name(h, b->core.tid), sam_hdr_tid2len(h, b->core.tid), &r)) {
+ fprintf(stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", sam_hdr_tid2name(h, b->core.tid));
return -1;
}
posmap = update_posmap(posmap, r);
r_tid = b->core.tid;
// fprintf(stderr, "[depad] Loaded %s from FASTA file\n", h->target_name[b->core.tid]);
} else {
- fprintf(stderr, "[depad] ERROR: Missing %s embedded reference sequence (and no FASTA file)\n", h->target_name[b->core.tid]);
+ fprintf(stderr, "[depad] ERROR: Missing %s embedded reference sequence (and no FASTA file)\n", sam_hdr_tid2name(h, b->core.tid));
return -1;
}
if (0!=unpad_seq(b, &q)) {
/* Nasty case, Must load alternative posmap */
// fprintf(stderr, "[depad] Loading reference '%s' temporarily\n", h->target_name[b->core.mtid]);
if (!fai) {
- fprintf(stderr, "[depad] ERROR: Needed reference %s sequence for mate (and no FASTA file)\n", h->target_name[b->core.mtid]);
+ fprintf(stderr, "[depad] ERROR: Needed reference %s sequence for mate (and no FASTA file)\n", sam_hdr_tid2name(h, b->core.mtid));
return -1;
}
/* Temporarily load the other reference sequence */
- if (load_unpadded_ref(fai, h->target_name[b->core.mtid], h->target_len[b->core.mtid], &r)) {
- fprintf(stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", h->target_name[b->core.mtid]);
+ if (load_unpadded_ref(fai, sam_hdr_tid2name(h, b->core.mtid), sam_hdr_tid2len(h, b->core.mtid), &r)) {
+ fprintf(stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", sam_hdr_tid2name(h, b->core.mtid));
return -1;
}
posmap = update_posmap(posmap, r);
b->core.mpos = posmap[b->core.mpos];
/* Restore the reference and posmap*/
- if (load_unpadded_ref(fai, h->target_name[b->core.tid], h->target_len[b->core.tid], &r)) {
- fprintf(stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", h->target_name[b->core.tid]);
+ if (load_unpadded_ref(fai, sam_hdr_tid2name(h, b->core.tid), sam_hdr_tid2len(h, b->core.tid), &r)) {
+ fprintf(stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", sam_hdr_tid2name(h, b->core.tid));
return -1;
}
posmap = update_posmap(posmap, r);
ret = 1;
}
free(r.s); free(q.s); free(posmap);
+ free(cigar2);
bam_destroy1(b);
return ret;
}
-bam_hdr_t * fix_header(bam_hdr_t *old, faidx_t *fai)
+sam_hdr_t * fix_header(sam_hdr_t *old, faidx_t *fai)
{
- int i = 0, unpadded_len = 0;
- bam_hdr_t *header = 0 ;
- unsigned short ln_found;
+ int i = 0, ret = 0;
+ hts_pos_t unpadded_len = 0;
+ sam_hdr_t *header = sam_hdr_dup(old);
+ if (!header)
+ return NULL;
- header = bam_hdr_dup(old);
- for (i = 0; i < old->n_targets; ++i) {
- unpadded_len = get_unpadded_len(fai, old->target_name[i], old->target_len[i]);
+ int nref = sam_hdr_nref(old);
+ char len_buf[64];
+
+ for (i = 0; i < nref; ++i) {
+ unpadded_len = get_unpadded_len(fai, sam_hdr_tid2name(old, i), sam_hdr_tid2len(old, i));
if (unpadded_len < 0) {
- fprintf(stderr, "[depad] ERROR getting unpadded length of '%s', padded length %i\n", old->target_name[i], old->target_len[i]);
+ fprintf(stderr, "[depad] ERROR getting unpadded length of '%s', padded length %"PRIhts_pos"\n", sam_hdr_tid2name(old, i), (hts_pos_t) sam_hdr_tid2len(old, i));
+ } else if (unpadded_len > sam_hdr_tid2len(old, i)) {
+ fprintf(stderr, "[depad] New unpadded length of '%s' is larger than the padded length (%"PRIhts_pos" > %"PRIhts_pos")\n",
+ sam_hdr_tid2name(old, i), unpadded_len,
+ (hts_pos_t) sam_hdr_tid2len(old, i));
+ ret = 1;
} else {
- header->target_len[i] = unpadded_len;
+ sprintf(len_buf, "%"PRIhts_pos"", unpadded_len);
+ if ((ret |= sam_hdr_update_line(header, "SQ", "SN", sam_hdr_tid2name(header, i), "LN", len_buf, NULL)))
+ fprintf(stderr, "[depad] Error updating length of '%s' from %"PRIhts_pos" to %"PRIhts_pos"\n",
+ sam_hdr_tid2name(header, i),
+ (hts_pos_t) sam_hdr_tid2len(header, i),
+ unpadded_len);
//fprintf(stderr, "[depad] Recalculating '%s' length %i -> %i\n", old->target_name[i], old->target_len[i], header->target_len[i]);
}
}
- /* Duplicating the header allocated new buffer for header string */
- /* After modifying the @SQ lines it will only get smaller, since */
- /* the LN entries will be the same or shorter, and we'll remove */
- /* any MD entries (MD5 checksums). */
- assert(strlen(old->text) == strlen(header->text));
- assert (0==strcmp(old->text, header->text));
- const char *text;
- text = old->text;
- header->text[0] = '\0'; /* Resuse the allocated buffer */
- char * newtext = header->text;
- char * end=NULL;
- while (text[0]=='@') {
- end = strchr(text, '\n');
- assert(end != 0);
- if (text[1]=='S' && text[2]=='Q' && text[3]=='\t') {
- const char *cp = text+3;
- char *name = strstr(text, "\tSN:");
- char *name_end;
- if (!name) {
- fprintf(stderr, "Unable to find SN: header field\n");
- return NULL;
- }
- name += 4;
- for (name_end = name; name_end != end && *name_end != '\t'; name_end++);
- strcat(newtext, "@SQ");
- ln_found = 0;
-
- /* Parse the @SQ lines */
- while (cp != end) {
- if (!ln_found && end-cp >= 2 && strncmp(cp, "LN", 2) == 0) {
- // Rewrite the length
- char len_buf[100];
- int tid;
- unsigned int old_length, new_length;
- const char *old_cp = cp;
-
- ln_found = 1;
-
- while (cp != end && *cp++ != '\t');
- old_length = (int)(cp - old_cp);
-
- for (tid = 0; tid < header->n_targets; tid++) {
- // may want to hash this, but new header API incoming.
- if (strncmp(name, header->target_name[tid], name_end - name) == 0) {
- new_length = sprintf(len_buf, "LN:%d", header->target_len[tid]);
- if (new_length <= old_length) {
- strcat(newtext, len_buf);
- }
- else {
- fprintf(stderr, "LN value of the reference is larger than the original!\n");
- exit(1);
- }
- break;
- }
- }
- if (cp != end)
- strcat(newtext, "\t");
- } else if (end-cp >= 2 &&
- ((ln_found && strncmp(cp, "LN", 2) == 0) ||
- strncmp(cp, "M5", 2) == 0 ||
- strncmp(cp, "UR", 2) == 0))
- {
- // skip secondary LNs
- // MD5 changed during depadding; ditch it.
- // URLs are also invalid.
- while (cp != end && *cp++ != '\t');
- } else {
- // Otherwise copy this sub-field verbatim
- const char *cp_start = cp;
- while (cp != end && *cp++ != '\t');
- strncat(newtext, cp_start, cp-cp_start);
- }
- }
-
- // Add newline, replacing trailing '\t' if last on line was the LN:
- char *text_end = newtext + strlen(newtext);
- if (text_end[-1] == '\t')
- text_end[-1] = '\n';
- else
- *text_end++ = '\n', *text_end = '\0';
- } else {
- /* Copy this line to the new header */
- strncat(newtext, text, end - text + 1);
- }
- text = end + 1;
+ if (ret) {
+ sam_hdr_destroy(header);
+ return NULL;
}
- assert (text[0]=='\0');
- /* Check we didn't overflow the buffer */
- assert (strlen(header->text) <= strlen(old->text));
- if (strlen(header->text) < header->l_text) {
- //fprintf(stderr, "[depad] Reallocating header buffer\n");
- assert (newtext == header->text);
- newtext = malloc(strlen(header->text) + 1);
- strcpy(newtext, header->text);
- free(header->text);
- header->text = newtext;
- header->l_text = strlen(newtext);
- }
- //fprintf(stderr, "[depad] Here is the new header (pending @SQ lines),\n\n%s\n(end)\n", header->text);
+
return header;
}
int main_pad2unpad(int argc, char *argv[])
{
samFile *in = 0, *out = 0;
- bam_hdr_t *h = 0, *h_fix = 0;
+ sam_hdr_t *h = 0, *h_fix = 0;
faidx_t *fai = 0;
- int c, compress_level = -1, is_long_help = 0;
- char in_mode[5], out_mode[6], *fn_out = 0, *fn_list = 0;
+ int c, compress_level = -1, is_long_help = 0, no_pg = 0;
+ char in_mode[5], out_mode[6], *fn_out = 0, *fn_list = 0, *fn_out_idx = NULL;
int ret=0;
+ char *arg_list = NULL;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 'T', '-'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
if (ga.out.format == unknown_format)
hts_parse_format(&ga.out, "bam");
break;
+ case 1: no_pg = 1; break;
case '?': is_long_help = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
fprintf(stderr, "[bam_fillmd] unrecognized option '-%c'\n\n", c);
goto depad_end;
}
if (fai) {
- h_fix = fix_header(h, fai);
+ if (!(h_fix = fix_header(h, fai))){
+ fprintf(stderr, "[depad] failed to fix the header from\n");
+ ret = 1;
+ goto depad_end;
+ }
} else {
fprintf(stderr, "[depad] Warning - reference lengths will not be corrected without FASTA reference\n");
h_fix = h;
if (ga.out.format == cram)
hts_set_opt(out, CRAM_OPT_NO_REF, 1);
+ if (!no_pg) {
+ if(!(arg_list = stringify_argv(argc+1, argv-1))) {
+ fprintf(stderr, "[depad] failed to create arg_list\n");
+ ret = 1;
+ goto depad_end;
+ }
+
+ if (sam_hdr_add_pg(h_fix, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ fprintf(stderr, "[depad] failed to add PG line to header\n");
+ ret = 1;
+ goto depad_end;
+ }
+ }
+
if (sam_hdr_write(out, h_fix) != 0) {
fprintf(stderr, "[depad] failed to write header.\n");
ret = 1;
goto depad_end;
}
+ if (ga.write_index) {
+ if (!(fn_out_idx = auto_index(out, fn_out, h_fix))) {
+ ret = 1;
+ goto depad_end;
+ }
+ }
// Do the depad
if (bam_pad2unpad(in, out, h, fai) != 0) ret = 1;
+ if (ga.write_index) {
+ if (sam_idx_save(out) < 0) {
+ print_error_errno("depad", "writing index failed");
+ ret = 1;
+ }
+ }
+
depad_end:
// close files, free and return
+ free(arg_list);
if (fai) fai_destroy(fai);
- if (h) bam_hdr_destroy(h);
+ if (h) sam_hdr_destroy(h);
+ if (h_fix && h_fix != h) sam_hdr_destroy(h_fix);
if (in) sam_close(in);
if (out && sam_close(out) < 0) {
fprintf(stderr, "[depad] error on closing output file.\n");
ret = 1;
}
free(fn_list); free(fn_out);
+ if (fn_out_idx)
+ free(fn_out_idx);
+ sam_global_args_free(&ga);
return ret;
}
fprintf(stderr, " -T, --reference FILE\n");
fprintf(stderr, " Padded reference sequence file [null]\n");
fprintf(stderr, " -o FILE Output file name [stdout]\n");
+ fprintf(stderr, " --no-PG do not add a PG line\n");
fprintf(stderr, " -? Longer help\n");
- sam_global_opt_help(stderr, "-...--");
+ sam_global_opt_help(stderr, "-...--..");
if (is_long_help)
fprintf(stderr,
/* padding.c -- depad subcommand.
Copyright (C) 2011, 2012 Broad Institute.
- Copyright (C) 2014-2016 Genome Research Ltd.
+ Copyright (C) 2014-2016, 2019 Genome Research Ltd.
Portions copyright (C) 2012, 2013 Peter Cock, The James Hutton Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include <string.h>
#include <assert.h>
#include <unistd.h>
+#include <inttypes.h>
#include <htslib/kstring.h>
#include <htslib/sam.h>
#include <htslib/faidx.h>
-#include "sam_header.h"
#include "sam_opts.h"
#include "samtools.h"
if (_n == _m) { \
_m = _m? _m<<1 : 4; \
_c = (uint32_t*)realloc(_c, _m * 4); \
+ if (!(_c)) { \
+ fprintf(samtools_stderr, "[depad] ERROR: Memory allocation failure.\n"); \
+ return -1; \
+ } \
} \
_c[_n++] = (_v); \
} while (0)
return length != s->l;
}
-int load_unpadded_ref(faidx_t *fai, char *ref_name, int ref_len, kstring_t *seq)
+int load_unpadded_ref(faidx_t *fai, const char *ref_name, hts_pos_t ref_len, kstring_t *seq)
{
char base;
char *fai_ref = 0;
- int fai_ref_len = 0, k;
+ hts_pos_t fai_ref_len = 0, k;
- fai_ref = fai_fetch(fai, ref_name, &fai_ref_len);
+ fai_ref = fai_fetch64(fai, ref_name, &fai_ref_len);
if (fai_ref_len != ref_len) {
- fprintf(samtools_stderr, "[depad] ERROR: FASTA sequence %s length %i, expected %i\n", ref_name, fai_ref_len, ref_len);
+ fprintf(samtools_stderr, "[depad] ERROR: FASTA sequence %s length %"PRIhts_pos", expected %"PRIhts_pos"\n", ref_name, fai_ref_len, ref_len);
free(fai_ref);
return -1;
}
return 0;
}
-int get_unpadded_len(faidx_t *fai, char *ref_name, int padded_len)
+hts_pos_t get_unpadded_len(faidx_t *fai, const char *ref_name, hts_pos_t padded_len)
{
char base;
char *fai_ref = 0;
- int fai_ref_len = 0, k;
- int bases=0, gaps=0;
+ hts_pos_t fai_ref_len = 0, k;
+ hts_pos_t bases=0, gaps=0;
- fai_ref = fai_fetch(fai, ref_name, &fai_ref_len);
+ fai_ref = fai_fetch64(fai, ref_name, &fai_ref_len);
if (fai_ref_len != padded_len) {
- fprintf(samtools_stderr, "[depad] ERROR: FASTA sequence '%s' length %i, expected %i\n", ref_name, fai_ref_len, padded_len);
+ fprintf(samtools_stderr, "[depad] ERROR: FASTA sequence '%s' length %"PRIhts_pos", expected %"PRIhts_pos"\n", ref_name, fai_ref_len, padded_len);
free(fai_ref);
return -1;
}
return posmap;
}
-int bam_pad2unpad(samFile *in, samFile *out, bam_hdr_t *h, faidx_t *fai)
+int bam_pad2unpad(samFile *in, samFile *out, sam_hdr_t *h, faidx_t *fai)
{
bam1_t *b = 0;
kstring_t r, q;
uint32_t *cigar = bam_get_cigar(b);
n2 = 0;
- if (b->core.pos == 0 && b->core.tid >= 0 && strcmp(bam_get_qname(b), h->target_name[b->core.tid]) == 0) {
+ if (b->core.pos == 0 && b->core.tid >= 0 && strcmp(bam_get_qname(b), sam_hdr_tid2name(h, b->core.tid)) == 0) {
// fprintf(samtools_stderr, "[depad] Found embedded reference '%s'\n", bam_get_qname(b));
r_tid = b->core.tid;
if (0!=unpad_seq(b, &r)) {
fprintf(samtools_stderr, "[depad] ERROR: Problem parsing SEQ and/or CIGAR in reference %s\n", bam_get_qname(b));
return -1;
};
- if (h->target_len[r_tid] != r.l) {
- fprintf(samtools_stderr, "[depad] ERROR: (Padded) length of '%s' is %u in BAM header, but %llu in embedded reference\n", bam_get_qname(b), h->target_len[r_tid], (unsigned long long)(r.l));
+ if (sam_hdr_tid2len(h, r_tid) != r.l) {
+ fprintf(samtools_stderr, "[depad] ERROR: (Padded) length of '%s' is %"PRId64" in BAM header, but %zu in embedded reference\n", bam_get_qname(b), (int64_t) sam_hdr_tid2len(h, r_tid), r.l);
return -1;
}
if (fai) {
// Check the embedded reference matches the FASTA file
- if (load_unpadded_ref(fai, h->target_name[b->core.tid], h->target_len[b->core.tid], &q)) {
- fprintf(samtools_stderr, "[depad] ERROR: Failed to load embedded reference '%s' from FASTA\n", h->target_name[b->core.tid]);
+ if (load_unpadded_ref(fai, sam_hdr_tid2name(h, b->core.tid), sam_hdr_tid2len(h, b->core.tid), &q)) {
+ fprintf(samtools_stderr, "[depad] ERROR: Failed to load embedded reference '%s' from FASTA\n", sam_hdr_tid2name(h, b->core.tid));
return -1;
}
assert(r.l == q.l);
if (r.s[i] != q.s[i]) {
// Show gaps as ASCII 45
fprintf(samtools_stderr, "[depad] ERROR: Embedded sequence and reference FASTA don't match for %s base %i, '%c' vs '%c'\n",
- h->target_name[b->core.tid], i+1,
+ sam_hdr_tid2name(h, b->core.tid), i+1,
r.s[i] ? seq_nt16_str[(int)r.s[i]] : 45,
q.s[i] ? seq_nt16_str[(int)q.s[i]] : 45);
return -1;
; // good case, reference available
//fprintf(samtools_stderr, "[depad] Have ref '%s' for read '%s'\n", h->target_name[b->core.tid], bam_get_qname(b));
} else if (fai) {
- if (load_unpadded_ref(fai, h->target_name[b->core.tid], h->target_len[b->core.tid], &r)) {
- fprintf(samtools_stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", h->target_name[b->core.tid]);
+ if (load_unpadded_ref(fai, sam_hdr_tid2name(h, b->core.tid), sam_hdr_tid2len(h, b->core.tid), &r)) {
+ fprintf(samtools_stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", sam_hdr_tid2name(h, b->core.tid));
return -1;
}
posmap = update_posmap(posmap, r);
r_tid = b->core.tid;
// fprintf(samtools_stderr, "[depad] Loaded %s from FASTA file\n", h->target_name[b->core.tid]);
} else {
- fprintf(samtools_stderr, "[depad] ERROR: Missing %s embedded reference sequence (and no FASTA file)\n", h->target_name[b->core.tid]);
+ fprintf(samtools_stderr, "[depad] ERROR: Missing %s embedded reference sequence (and no FASTA file)\n", sam_hdr_tid2name(h, b->core.tid));
return -1;
}
if (0!=unpad_seq(b, &q)) {
/* Nasty case, Must load alternative posmap */
// fprintf(samtools_stderr, "[depad] Loading reference '%s' temporarily\n", h->target_name[b->core.mtid]);
if (!fai) {
- fprintf(samtools_stderr, "[depad] ERROR: Needed reference %s sequence for mate (and no FASTA file)\n", h->target_name[b->core.mtid]);
+ fprintf(samtools_stderr, "[depad] ERROR: Needed reference %s sequence for mate (and no FASTA file)\n", sam_hdr_tid2name(h, b->core.mtid));
return -1;
}
/* Temporarily load the other reference sequence */
- if (load_unpadded_ref(fai, h->target_name[b->core.mtid], h->target_len[b->core.mtid], &r)) {
- fprintf(samtools_stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", h->target_name[b->core.mtid]);
+ if (load_unpadded_ref(fai, sam_hdr_tid2name(h, b->core.mtid), sam_hdr_tid2len(h, b->core.mtid), &r)) {
+ fprintf(samtools_stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", sam_hdr_tid2name(h, b->core.mtid));
return -1;
}
posmap = update_posmap(posmap, r);
b->core.mpos = posmap[b->core.mpos];
/* Restore the reference and posmap*/
- if (load_unpadded_ref(fai, h->target_name[b->core.tid], h->target_len[b->core.tid], &r)) {
- fprintf(samtools_stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", h->target_name[b->core.tid]);
+ if (load_unpadded_ref(fai, sam_hdr_tid2name(h, b->core.tid), sam_hdr_tid2len(h, b->core.tid), &r)) {
+ fprintf(samtools_stderr, "[depad] ERROR: Failed to load '%s' from reference FASTA\n", sam_hdr_tid2name(h, b->core.tid));
return -1;
}
posmap = update_posmap(posmap, r);
ret = 1;
}
free(r.s); free(q.s); free(posmap);
+ free(cigar2);
bam_destroy1(b);
return ret;
}
-bam_hdr_t * fix_header(bam_hdr_t *old, faidx_t *fai)
+sam_hdr_t * fix_header(sam_hdr_t *old, faidx_t *fai)
{
- int i = 0, unpadded_len = 0;
- bam_hdr_t *header = 0 ;
- unsigned short ln_found;
+ int i = 0, ret = 0;
+ hts_pos_t unpadded_len = 0;
+ sam_hdr_t *header = sam_hdr_dup(old);
+ if (!header)
+ return NULL;
- header = bam_hdr_dup(old);
- for (i = 0; i < old->n_targets; ++i) {
- unpadded_len = get_unpadded_len(fai, old->target_name[i], old->target_len[i]);
+ int nref = sam_hdr_nref(old);
+ char len_buf[64];
+
+ for (i = 0; i < nref; ++i) {
+ unpadded_len = get_unpadded_len(fai, sam_hdr_tid2name(old, i), sam_hdr_tid2len(old, i));
if (unpadded_len < 0) {
- fprintf(samtools_stderr, "[depad] ERROR getting unpadded length of '%s', padded length %i\n", old->target_name[i], old->target_len[i]);
+ fprintf(samtools_stderr, "[depad] ERROR getting unpadded length of '%s', padded length %"PRIhts_pos"\n", sam_hdr_tid2name(old, i), (hts_pos_t) sam_hdr_tid2len(old, i));
+ } else if (unpadded_len > sam_hdr_tid2len(old, i)) {
+ fprintf(samtools_stderr, "[depad] New unpadded length of '%s' is larger than the padded length (%"PRIhts_pos" > %"PRIhts_pos")\n",
+ sam_hdr_tid2name(old, i), unpadded_len,
+ (hts_pos_t) sam_hdr_tid2len(old, i));
+ ret = 1;
} else {
- header->target_len[i] = unpadded_len;
+ sprintf(len_buf, "%"PRIhts_pos"", unpadded_len);
+ if ((ret |= sam_hdr_update_line(header, "SQ", "SN", sam_hdr_tid2name(header, i), "LN", len_buf, NULL)))
+ fprintf(samtools_stderr, "[depad] Error updating length of '%s' from %"PRIhts_pos" to %"PRIhts_pos"\n",
+ sam_hdr_tid2name(header, i),
+ (hts_pos_t) sam_hdr_tid2len(header, i),
+ unpadded_len);
//fprintf(samtools_stderr, "[depad] Recalculating '%s' length %i -> %i\n", old->target_name[i], old->target_len[i], header->target_len[i]);
}
}
- /* Duplicating the header allocated new buffer for header string */
- /* After modifying the @SQ lines it will only get smaller, since */
- /* the LN entries will be the same or shorter, and we'll remove */
- /* any MD entries (MD5 checksums). */
- assert(strlen(old->text) == strlen(header->text));
- assert (0==strcmp(old->text, header->text));
- const char *text;
- text = old->text;
- header->text[0] = '\0'; /* Resuse the allocated buffer */
- char * newtext = header->text;
- char * end=NULL;
- while (text[0]=='@') {
- end = strchr(text, '\n');
- assert(end != 0);
- if (text[1]=='S' && text[2]=='Q' && text[3]=='\t') {
- const char *cp = text+3;
- char *name = strstr(text, "\tSN:");
- char *name_end;
- if (!name) {
- fprintf(samtools_stderr, "Unable to find SN: header field\n");
- return NULL;
- }
- name += 4;
- for (name_end = name; name_end != end && *name_end != '\t'; name_end++);
- strcat(newtext, "@SQ");
- ln_found = 0;
-
- /* Parse the @SQ lines */
- while (cp != end) {
- if (!ln_found && end-cp >= 2 && strncmp(cp, "LN", 2) == 0) {
- // Rewrite the length
- char len_buf[100];
- int tid;
- unsigned int old_length, new_length;
- const char *old_cp = cp;
-
- ln_found = 1;
-
- while (cp != end && *cp++ != '\t');
- old_length = (int)(cp - old_cp);
-
- for (tid = 0; tid < header->n_targets; tid++) {
- // may want to hash this, but new header API incoming.
- if (strncmp(name, header->target_name[tid], name_end - name) == 0) {
- new_length = sprintf(len_buf, "LN:%d", header->target_len[tid]);
- if (new_length <= old_length) {
- strcat(newtext, len_buf);
- }
- else {
- fprintf(samtools_stderr, "LN value of the reference is larger than the original!\n");
- exit(1);
- }
- break;
- }
- }
- if (cp != end)
- strcat(newtext, "\t");
- } else if (end-cp >= 2 &&
- ((ln_found && strncmp(cp, "LN", 2) == 0) ||
- strncmp(cp, "M5", 2) == 0 ||
- strncmp(cp, "UR", 2) == 0))
- {
- // skip secondary LNs
- // MD5 changed during depadding; ditch it.
- // URLs are also invalid.
- while (cp != end && *cp++ != '\t');
- } else {
- // Otherwise copy this sub-field verbatim
- const char *cp_start = cp;
- while (cp != end && *cp++ != '\t');
- strncat(newtext, cp_start, cp-cp_start);
- }
- }
-
- // Add newline, replacing trailing '\t' if last on line was the LN:
- char *text_end = newtext + strlen(newtext);
- if (text_end[-1] == '\t')
- text_end[-1] = '\n';
- else
- *text_end++ = '\n', *text_end = '\0';
- } else {
- /* Copy this line to the new header */
- strncat(newtext, text, end - text + 1);
- }
- text = end + 1;
+ if (ret) {
+ sam_hdr_destroy(header);
+ return NULL;
}
- assert (text[0]=='\0');
- /* Check we didn't overflow the buffer */
- assert (strlen(header->text) <= strlen(old->text));
- if (strlen(header->text) < header->l_text) {
- //fprintf(samtools_stderr, "[depad] Reallocating header buffer\n");
- assert (newtext == header->text);
- newtext = malloc(strlen(header->text) + 1);
- strcpy(newtext, header->text);
- free(header->text);
- header->text = newtext;
- header->l_text = strlen(newtext);
- }
- //fprintf(samtools_stderr, "[depad] Here is the new header (pending @SQ lines),\n\n%s\n(end)\n", header->text);
+
return header;
}
int main_pad2unpad(int argc, char *argv[])
{
samFile *in = 0, *out = 0;
- bam_hdr_t *h = 0, *h_fix = 0;
+ sam_hdr_t *h = 0, *h_fix = 0;
faidx_t *fai = 0;
- int c, compress_level = -1, is_long_help = 0;
- char in_mode[5], out_mode[6], *fn_out = 0, *fn_list = 0;
+ int c, compress_level = -1, is_long_help = 0, no_pg = 0;
+ char in_mode[5], out_mode[6], *fn_out = 0, *fn_list = 0, *fn_out_idx = NULL;
int ret=0;
+ char *arg_list = NULL;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 'T', '-'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
if (ga.out.format == unknown_format)
hts_parse_format(&ga.out, "bam");
break;
+ case 1: no_pg = 1; break;
case '?': is_long_help = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
fprintf(samtools_stderr, "[bam_fillmd] unrecognized option '-%c'\n\n", c);
goto depad_end;
}
if (fai) {
- h_fix = fix_header(h, fai);
+ if (!(h_fix = fix_header(h, fai))){
+ fprintf(samtools_stderr, "[depad] failed to fix the header from\n");
+ ret = 1;
+ goto depad_end;
+ }
} else {
fprintf(samtools_stderr, "[depad] Warning - reference lengths will not be corrected without FASTA reference\n");
h_fix = h;
if (ga.out.format == cram)
hts_set_opt(out, CRAM_OPT_NO_REF, 1);
+ if (!no_pg) {
+ if(!(arg_list = stringify_argv(argc+1, argv-1))) {
+ fprintf(samtools_stderr, "[depad] failed to create arg_list\n");
+ ret = 1;
+ goto depad_end;
+ }
+
+ if (sam_hdr_add_pg(h_fix, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ fprintf(samtools_stderr, "[depad] failed to add PG line to header\n");
+ ret = 1;
+ goto depad_end;
+ }
+ }
+
if (sam_hdr_write(out, h_fix) != 0) {
fprintf(samtools_stderr, "[depad] failed to write header.\n");
ret = 1;
goto depad_end;
}
+ if (ga.write_index) {
+ if (!(fn_out_idx = auto_index(out, fn_out, h_fix))) {
+ ret = 1;
+ goto depad_end;
+ }
+ }
// Do the depad
if (bam_pad2unpad(in, out, h, fai) != 0) ret = 1;
+ if (ga.write_index) {
+ if (sam_idx_save(out) < 0) {
+ print_error_errno("depad", "writing index failed");
+ ret = 1;
+ }
+ }
+
depad_end:
// close files, free and return
+ free(arg_list);
if (fai) fai_destroy(fai);
- if (h) bam_hdr_destroy(h);
+ if (h) sam_hdr_destroy(h);
+ if (h_fix && h_fix != h) sam_hdr_destroy(h_fix);
if (in) sam_close(in);
if (out && sam_close(out) < 0) {
fprintf(samtools_stderr, "[depad] error on closing output file.\n");
ret = 1;
}
free(fn_list); free(fn_out);
+ if (fn_out_idx)
+ free(fn_out_idx);
+ sam_global_args_free(&ga);
return ret;
}
fprintf(samtools_stderr, " -T, --reference FILE\n");
fprintf(samtools_stderr, " Padded reference sequence file [null]\n");
fprintf(samtools_stderr, " -o FILE Output file name [samtools_stdout]\n");
+ fprintf(samtools_stderr, " --no-PG do not add a PG line\n");
fprintf(samtools_stderr, " -? Longer help\n");
- sam_global_opt_help(samtools_stderr, "-...--");
+ sam_global_opt_help(samtools_stderr, "-...--..");
if (is_long_help)
fprintf(samtools_stderr,
/* phase.c -- phase subcommand.
Copyright (C) 2011 Broad Institute.
- Copyright (C) 2013-2016 Genome Research Ltd.
+ Copyright (C) 2013-2016, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
typedef struct {
// configurations, initialized in the main function
- int flag, k, min_baseQ, min_varLOD, max_depth;
+ int flag, k, min_baseQ, min_varLOD, max_depth, no_pg;
// other global variables
int vpos_shift;
samFile* fp;
- bam_hdr_t* fp_hdr;
- char *pre;
+ sam_hdr_t* fp_hdr;
+ char *pre, *arg_list;
char *out_name[3];
samFile* out[3];
- bam_hdr_t* out_hdr[3];
+ sam_hdr_t* out_hdr[3];
// alignment queue
int n, m;
bam1_t **b;
return ret;
}
-static khash_t(set64) *loadpos(const char *fn, bam_hdr_t *h)
+static khash_t(set64) *loadpos(const char *fn, sam_hdr_t *h)
{
gzFile fp;
kstream_t *ks;
kstring_t *str;
khash_t(set64) *hash;
+ fp = strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
+ if (fp == NULL) {
+ print_error_errno("phase", "Couldn't open site file '%s'", fn);
+ return NULL;
+ }
+
hash = kh_init(set64);
str = calloc(1, sizeof(kstring_t));
- fp = strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
+
ks = ks_init(fp);
while (ks_getuntil(ks, 0, str, &dret) >= 0) {
int tid = bam_name2id(h, str->s);
return -1;
}
- g->out_hdr[c] = bam_hdr_dup(g->fp_hdr);
+ g->out_hdr[c] = sam_hdr_dup(g->fp_hdr);
+ if (!g->no_pg && sam_hdr_add_pg(g->out_hdr[c], "samtools",
+ "VN", samtools_version(),
+ g->arg_list ? "CL": NULL,
+ g->arg_list ? g->arg_list : NULL,
+ NULL)) {
+ print_error("phase", "failed to add PG line to header");
+ return -1;
+ }
if (sam_hdr_write(g->out[c], g->out_hdr[c]) < 0) {
print_error_errno("phase", "Failed to write header for '%s'", g->out_name[c]);
return -1;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 0, '-'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
case 'A': g.flag |= FLAG_DROP_AMBI; break;
case 'b': g.pre = strdup(optarg); break;
case 'l': fn_list = strdup(optarg); break;
+ case 1: g.no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': usage=1; break;
// fprintf(stderr, " -l FILE list of sites to phase [null]\n");
fprintf(stderr, " -F do not attempt to fix chimeras\n");
fprintf(stderr, " -A drop reads with ambiguous phase\n");
+ fprintf(stderr, " --no-PG do not add a PG line\n");
// fprintf(stderr, " -e do not discover SNPs (effective with -l)\n");
fprintf(stderr, "\n");
- sam_global_opt_help(stderr, "-....-");
+ sam_global_opt_help(stderr, "-....--.");
return 1;
}
__func__, argv[optind]);
return 1;
}
+ if (!g.no_pg && !(g.arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("phase", "failed to create arg_list");
+ return 1;
+ }
if (fn_list) { // read the list of sites to phase
set = loadpos(fn_list, g.fp_hdr);
+ if (set == NULL) return 1;
free(fn_list);
} else g.flag &= ~FLAG_LIST_EXCL;
if (g.pre) { // open BAMs to write
g.vpos_shift = 0;
if (lasttid >= 0) {
seqs = shrink_hash(seqs);
- if (phase(&g, g.fp_hdr->target_name[lasttid],
+ if (phase(&g, sam_hdr_tid2name(g.fp_hdr, lasttid),
vpos, cns, seqs) < 0) {
return 1;
}
}
if (dophase) {
seqs = shrink_hash(seqs);
- if (phase(&g, g.fp_hdr->target_name[tid], vpos, cns, seqs) < 0) {
+ if (phase(&g, sam_hdr_tid2name(g.fp_hdr, tid), vpos, cns, seqs) < 0) {
return 1;
}
update_vpos(vpos, seqs);
++vpos;
}
if (tid >= 0) {
- if (phase(&g, g.fp_hdr->target_name[tid], vpos, cns, seqs) < 0) {
+ if (phase(&g, sam_hdr_tid2name(g.fp_hdr, tid), vpos, cns, seqs) < 0) {
return 1;
}
}
- bam_hdr_destroy(g.fp_hdr);
+ sam_hdr_destroy(g.fp_hdr);
bam_plp_destroy(iter);
sam_close(g.fp);
kh_destroy(64, seqs);
__func__, g.out_name[c]);
res = 1;
}
- bam_hdr_destroy(g.out_hdr[c]);
+ sam_hdr_destroy(g.out_hdr[c]);
free(g.out_name[c]);
}
free(g.pre); free(g.b);
if (res) return 1;
}
+ free(g.arg_list);
sam_global_args_free(&ga);
return 0;
}
/* phase.c -- phase subcommand.
Copyright (C) 2011 Broad Institute.
- Copyright (C) 2013-2016 Genome Research Ltd.
+ Copyright (C) 2013-2016, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
typedef struct {
// configurations, initialized in the main function
- int flag, k, min_baseQ, min_varLOD, max_depth;
+ int flag, k, min_baseQ, min_varLOD, max_depth, no_pg;
// other global variables
int vpos_shift;
samFile* fp;
- bam_hdr_t* fp_hdr;
- char *pre;
+ sam_hdr_t* fp_hdr;
+ char *pre, *arg_list;
char *out_name[3];
samFile* out[3];
- bam_hdr_t* out_hdr[3];
+ sam_hdr_t* out_hdr[3];
// alignment queue
int n, m;
bam1_t **b;
return ret;
}
-static khash_t(set64) *loadpos(const char *fn, bam_hdr_t *h)
+static khash_t(set64) *loadpos(const char *fn, sam_hdr_t *h)
{
gzFile fp;
kstream_t *ks;
kstring_t *str;
khash_t(set64) *hash;
+ fp = strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
+ if (fp == NULL) {
+ print_error_errno("phase", "Couldn't open site file '%s'", fn);
+ return NULL;
+ }
+
hash = kh_init(set64);
str = calloc(1, sizeof(kstring_t));
- fp = strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
+
ks = ks_init(fp);
while (ks_getuntil(ks, 0, str, &dret) >= 0) {
int tid = bam_name2id(h, str->s);
return -1;
}
- g->out_hdr[c] = bam_hdr_dup(g->fp_hdr);
+ g->out_hdr[c] = sam_hdr_dup(g->fp_hdr);
+ if (!g->no_pg && sam_hdr_add_pg(g->out_hdr[c], "samtools",
+ "VN", samtools_version(),
+ g->arg_list ? "CL": NULL,
+ g->arg_list ? g->arg_list : NULL,
+ NULL)) {
+ print_error("phase", "failed to add PG line to header");
+ return -1;
+ }
if (sam_hdr_write(g->out[c], g->out_hdr[c]) < 0) {
print_error_errno("phase", "Failed to write header for '%s'", g->out_name[c]);
return -1;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 0, 0, 0, '-'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
case 'A': g.flag |= FLAG_DROP_AMBI; break;
case 'b': g.pre = strdup(optarg); break;
case 'l': fn_list = strdup(optarg); break;
+ case 1: g.no_pg = 1; break;
default: if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
/* else fall-through */
case '?': usage=1; break;
// fprintf(samtools_stderr, " -l FILE list of sites to phase [null]\n");
fprintf(samtools_stderr, " -F do not attempt to fix chimeras\n");
fprintf(samtools_stderr, " -A drop reads with ambiguous phase\n");
+ fprintf(samtools_stderr, " --no-PG do not add a PG line\n");
// fprintf(samtools_stderr, " -e do not discover SNPs (effective with -l)\n");
fprintf(samtools_stderr, "\n");
- sam_global_opt_help(samtools_stderr, "-....-");
+ sam_global_opt_help(samtools_stderr, "-....--.");
return 1;
}
__func__, argv[optind]);
return 1;
}
+ if (!g.no_pg && !(g.arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("phase", "failed to create arg_list");
+ return 1;
+ }
if (fn_list) { // read the list of sites to phase
set = loadpos(fn_list, g.fp_hdr);
+ if (set == NULL) return 1;
free(fn_list);
} else g.flag &= ~FLAG_LIST_EXCL;
if (g.pre) { // open BAMs to write
g.vpos_shift = 0;
if (lasttid >= 0) {
seqs = shrink_hash(seqs);
- if (phase(&g, g.fp_hdr->target_name[lasttid],
+ if (phase(&g, sam_hdr_tid2name(g.fp_hdr, lasttid),
vpos, cns, seqs) < 0) {
return 1;
}
}
if (dophase) {
seqs = shrink_hash(seqs);
- if (phase(&g, g.fp_hdr->target_name[tid], vpos, cns, seqs) < 0) {
+ if (phase(&g, sam_hdr_tid2name(g.fp_hdr, tid), vpos, cns, seqs) < 0) {
return 1;
}
update_vpos(vpos, seqs);
++vpos;
}
if (tid >= 0) {
- if (phase(&g, g.fp_hdr->target_name[tid], vpos, cns, seqs) < 0) {
+ if (phase(&g, sam_hdr_tid2name(g.fp_hdr, tid), vpos, cns, seqs) < 0) {
return 1;
}
}
- bam_hdr_destroy(g.fp_hdr);
+ sam_hdr_destroy(g.fp_hdr);
bam_plp_destroy(iter);
sam_close(g.fp);
kh_destroy(64, seqs);
__func__, g.out_name[c]);
res = 1;
}
- bam_hdr_destroy(g.out_hdr[c]);
+ sam_hdr_destroy(g.out_hdr[c]);
free(g.out_name[c]);
}
free(g.pre); free(g.b);
if (res) return 1;
}
+ free(g.arg_list);
sam_global_args_free(&ga);
return 0;
}
/* sam.c -- format-neutral SAM/BAM API.
- Copyright (C) 2009, 2012-2015 Genome Research Ltd.
+ Copyright (C) 2009, 2012-2016 Genome Research Ltd.
Portions copyright (C) 2011 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
return NULL;
}
fp->is_write = 0;
- if (fp->header->n_targets == 0 && bam_verbose >= 1)
+ if (sam_hdr_nref(fp->header) == 0 && bam_verbose >= 1)
fprintf(stderr, "[samopen] no @SQ lines in the header.\n");
}
else {
enum htsExactFormat fmt = hts_get_format(fp->file)->format;
- fp->header = (bam_hdr_t *)aux; // For writing, we won't free it
+ fp->header = (sam_hdr_t *)aux; // For writing, we won't free it
fp->is_write = 1;
if (!(fmt == text_format || fmt == sam) || strchr(mode, 'h')) {
if (sam_hdr_write(fp->file, fp->header) < 0) {
void samclose(samfile_t *fp)
{
if (fp) {
- if (!fp->is_write && fp->header) bam_hdr_destroy(fp->header);
+ if (!fp->is_write && fp->header) sam_hdr_destroy(fp->header);
sam_close(fp->file);
free(fp);
}
/* sam.c -- format-neutral SAM/BAM API.
- Copyright (C) 2009, 2012-2015 Genome Research Ltd.
+ Copyright (C) 2009, 2012-2016 Genome Research Ltd.
Portions copyright (C) 2011 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
return NULL;
}
fp->is_write = 0;
- if (fp->header->n_targets == 0 && bam_verbose >= 1)
+ if (sam_hdr_nref(fp->header) == 0 && bam_verbose >= 1)
fprintf(samtools_stderr, "[samopen] no @SQ lines in the header.\n");
}
else {
enum htsExactFormat fmt = hts_get_format(fp->file)->format;
- fp->header = (bam_hdr_t *)aux; // For writing, we won't free it
+ fp->header = (sam_hdr_t *)aux; // For writing, we won't free it
fp->is_write = 1;
if (!(fmt == text_format || fmt == sam) || strchr(mode, 'h')) {
if (sam_hdr_write(fp->file, fp->header) < 0) {
void samclose(samfile_t *fp)
{
if (fp) {
- if (!fp->is_write && fp->header) bam_hdr_destroy(fp->header);
+ if (!fp->is_write && fp->header) sam_hdr_destroy(fp->header);
sam_close(fp->file);
free(fp);
}
/* sam.h -- format-neutral SAM/BAM API.
- Copyright (C) 2009, 2013-2015 Genome Research Ltd.
+ Copyright (C) 2009, 2013-2015, 2019 Genome Research Ltd.
Author: Heng Li <lh3@sanger.ac.uk>
typedef struct {
samFile *file;
struct { BGZF *bam; } x; // Hack so that fp->x.bam still works
- bam_hdr_t *header;
+ sam_hdr_t *header;
unsigned short is_write:1;
} samfile_t;
static inline int samwrite(samfile_t *fp, const bam1_t *b) { return sam_write1(fp->file, fp->header, b); }
/*!
- @abstract Load BAM/CRAM index for use with samfetch()
+ @abstract Load BAM/CRAM index for use with samfetch() with supporting the use of index file
@param fp file handler
@param fn name of the BAM or CRAM file (NOT the index file)
+ @param fnidx name of the index file
@return pointer to the index structure
*/
- static inline bam_index_t *samtools_sam_index_load(samfile_t *fp, const char *fn) { return sam_index_load(fp->file, fn); }
+ static inline bam_index_t *samtools_sam_index_load(samfile_t *fp, const char *fn, const char *fnidx) {
+ if (fnidx != NULL) {
+ return sam_index_load2(fp->file, fn, fnidx);
+ }
+ return sam_index_load(fp->file, fn);
+ }
#undef sam_index_load
- #define sam_index_load(fp,fn) (samtools_sam_index_load((fp), (fn)))
+ #define sam_index_load(fp,fn,fnidx) (samtools_sam_index_load((fp), (fn), (fnidx)))
/*!
@abstract Retrieve the alignments overlapping the specified region.
+++ /dev/null
-/* sam_header.c -- basic SAM/BAM header API.
-
- Copyright (C) 2009-2013 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "sam_header.h"
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
-#include <stdlib.h>
-#include <stdarg.h>
-
-#include "htslib/khash.h"
-KHASH_MAP_INIT_STR(str, const char *)
-
-struct _HeaderList
-{
- struct _HeaderList *last; // Hack: Used and maintained only by list_append_to_end. Maintained in the root node only.
- struct _HeaderList *next;
- void *data;
-};
-typedef struct _HeaderList list_t;
-typedef list_t HeaderDict;
-
-typedef struct
-{
- char key[2];
- char *value;
-}
-HeaderTag;
-
-typedef struct
-{
- char type[2];
- list_t *tags;
-}
-HeaderLine;
-
-const char *o_hd_tags[] = {"SO","GO",NULL};
-const char *r_hd_tags[] = {"VN",NULL};
-
-const char *o_sq_tags[] = {"AS","M5","UR","SP",NULL};
-const char *r_sq_tags[] = {"SN","LN",NULL};
-const char *u_sq_tags[] = {"SN",NULL};
-
-const char *o_rg_tags[] = {"CN","DS","DT","FO","KS","LB","PG","PI","PL","PU","SM",NULL};
-const char *r_rg_tags[] = {"ID",NULL};
-const char *u_rg_tags[] = {"ID",NULL};
-
-const char *o_pg_tags[] = {"VN","CL",NULL};
-const char *r_pg_tags[] = {"ID",NULL};
-
-const char *types[] = {"HD","SQ","RG","PG","CO",NULL};
-const char **optional_tags[] = {o_hd_tags,o_sq_tags,o_rg_tags,o_pg_tags,NULL,NULL};
-const char **required_tags[] = {r_hd_tags,r_sq_tags,r_rg_tags,r_pg_tags,NULL,NULL};
-const char **unique_tags[] = {NULL, u_sq_tags,u_rg_tags,NULL,NULL,NULL};
-
-
-static void debug(const char *format, ...)
-{
- va_list ap;
- va_start(ap, format);
- vfprintf(stderr, format, ap);
- va_end(ap);
-}
-
-#if 0
-// Replaced by list_append_to_end
-static list_t *list_prepend(list_t *root, void *data)
-{
- list_t *l = malloc(sizeof(list_t));
- l->next = root;
- l->data = data;
- return l;
-}
-#endif
-
-// Relies on the root->last being correct. Do not use with the other list_*
-// routines unless they are fixed to modify root->last as well.
-static list_t *list_append_to_end(list_t *root, void *data)
-{
- list_t *l = malloc(sizeof(list_t));
- l->last = l;
- l->next = NULL;
- l->data = data;
-
- if ( !root )
- return l;
-
- root->last->next = l;
- root->last = l;
- return root;
-}
-
-static list_t *list_append(list_t *root, void *data)
-{
- list_t *l = root;
- while (l && l->next)
- l = l->next;
- if ( l )
- {
- l->next = malloc(sizeof(list_t));
- l = l->next;
- }
- else
- {
- l = malloc(sizeof(list_t));
- root = l;
- }
- l->data = data;
- l->next = NULL;
- return root;
-}
-
-static void list_free(list_t *root)
-{
- list_t *l = root;
- while (root)
- {
- l = root;
- root = root->next;
- free(l);
- }
-}
-
-
-
-// Look for a tag "XY" in a predefined const char *[] array.
-static int tag_exists(const char *tag, const char **tags)
-{
- int itag=0;
- if ( !tags ) return -1;
- while ( tags[itag] )
- {
- if ( tags[itag][0]==tag[0] && tags[itag][1]==tag[1] ) return itag;
- itag++;
- }
- return -1;
-}
-
-
-
-// Mimics the behaviour of getline, except it returns pointer to the next chunk of the text
-// or NULL if everything has been read. The lineptr should be freed by the caller. The
-// newline character is stripped.
-static const char *nextline(char **lineptr, size_t *n, const char *text)
-{
- int len;
- const char *to = text;
-
- if ( !*to ) return NULL;
-
- while ( *to && *to!='\n' && *to!='\r' ) to++;
- len = to - text + 1;
-
- if ( *to )
- {
- // Advance the pointer for the next call
- if ( *to=='\n' ) to++;
- else if ( *to=='\r' && *(to+1)=='\n' ) to+=2;
- }
- if ( !len )
- return to;
-
- if ( !*lineptr )
- {
- *lineptr = malloc(len);
- *n = len;
- }
- else if ( *n<len )
- {
- *lineptr = realloc(*lineptr, len);
- *n = len;
- }
- if ( !*lineptr ) {
- debug("[nextline] Insufficient memory!\n");
- return 0;
- }
-
- memcpy(*lineptr,text,len);
- (*lineptr)[len-1] = 0;
-
- return to;
-}
-
-// name points to "XY", value_from points to the first character of the value string and
-// value_to points to the last character of the value string.
-static HeaderTag *new_tag(const char *name, const char *value_from, const char *value_to)
-{
- HeaderTag *tag = malloc(sizeof(HeaderTag));
- int len = value_to-value_from+1;
-
- tag->key[0] = name[0];
- tag->key[1] = name[1];
- tag->value = malloc(len+1);
- memcpy(tag->value,value_from,len+1);
- tag->value[len] = 0;
- return tag;
-}
-
-static HeaderTag *header_line_has_tag(HeaderLine *hline, const char *key)
-{
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- if ( tag->key[0]==key[0] && tag->key[1]==key[1] ) return tag;
- tags = tags->next;
- }
- return NULL;
-}
-
-
-// Return codes:
-// 0 .. different types or unique tags differ or conflicting tags, cannot be merged
-// 1 .. all tags identical -> no need to merge, drop one
-// 2 .. the unique tags match and there are some conflicting tags (same tag, different value) -> error, cannot be merged nor duplicated
-// 3 .. there are some missing complementary tags and no unique conflict -> can be merged into a single line
-static int sam_header_compare_lines(HeaderLine *hline1, HeaderLine *hline2)
-{
- HeaderTag *t1, *t2;
-
- if ( hline1->type[0]!=hline2->type[0] || hline1->type[1]!=hline2->type[1] )
- return 0;
-
- int itype = tag_exists(hline1->type,types);
- if ( itype==-1 ) {
- debug("[sam_header_compare_lines] Unknown type [%c%c]\n", hline1->type[0],hline1->type[1]);
- return -1; // FIXME (lh3): error; I do not know how this will be handled in Petr's code
- }
-
- if ( unique_tags[itype] )
- {
- t1 = header_line_has_tag(hline1,unique_tags[itype][0]);
- t2 = header_line_has_tag(hline2,unique_tags[itype][0]);
- if ( !t1 || !t2 ) // this should never happen, the unique tags are required
- return 2;
-
- if ( strcmp(t1->value,t2->value) )
- return 0; // the unique tags differ, cannot be merged
- }
- if ( !required_tags[itype] && !optional_tags[itype] )
- {
- t1 = hline1->tags->data;
- t2 = hline2->tags->data;
- if ( !strcmp(t1->value,t2->value) ) return 1; // identical comments
- return 0;
- }
-
- int missing=0, itag=0;
- while ( required_tags[itype] && required_tags[itype][itag] )
- {
- t1 = header_line_has_tag(hline1,required_tags[itype][itag]);
- t2 = header_line_has_tag(hline2,required_tags[itype][itag]);
- if ( !t1 && !t2 )
- return 2; // this should never happen
- else if ( !t1 || !t2 )
- missing = 1; // there is some tag missing in one of the hlines
- else if ( strcmp(t1->value,t2->value) )
- {
- if ( unique_tags[itype] )
- return 2; // the lines have a matching unique tag but have a conflicting tag
-
- return 0; // the lines contain conflicting tags, cannot be merged
- }
- itag++;
- }
- itag = 0;
- while ( optional_tags[itype] && optional_tags[itype][itag] )
- {
- t1 = header_line_has_tag(hline1,optional_tags[itype][itag]);
- t2 = header_line_has_tag(hline2,optional_tags[itype][itag]);
- if ( !t1 && !t2 )
- {
- itag++;
- continue;
- }
- if ( !t1 || !t2 )
- missing = 1; // there is some tag missing in one of the hlines
- else if ( strcmp(t1->value,t2->value) )
- {
- if ( unique_tags[itype] )
- return 2; // the lines have a matching unique tag but have a conflicting tag
-
- return 0; // the lines contain conflicting tags, cannot be merged
- }
- itag++;
- }
- if ( missing ) return 3; // there are some missing complementary tags with no conflicts, can be merged
- return 1;
-}
-
-
-static HeaderLine *sam_header_line_clone(const HeaderLine *hline)
-{
- list_t *tags;
- HeaderLine *out = malloc(sizeof(HeaderLine));
- out->type[0] = hline->type[0];
- out->type[1] = hline->type[1];
- out->tags = NULL;
-
- tags = hline->tags;
- while (tags)
- {
- HeaderTag *old = tags->data;
-
- HeaderTag *new = malloc(sizeof(HeaderTag));
- new->key[0] = old->key[0];
- new->key[1] = old->key[1];
- new->value = strdup(old->value);
- out->tags = list_append(out->tags, new);
-
- tags = tags->next;
- }
- return out;
-}
-
-static int sam_header_line_merge_with(HeaderLine *out_hline, const HeaderLine *tmpl_hline)
-{
- list_t *tmpl_tags;
-
- if ( out_hline->type[0]!=tmpl_hline->type[0] || out_hline->type[1]!=tmpl_hline->type[1] )
- return 0;
-
- tmpl_tags = tmpl_hline->tags;
- while (tmpl_tags)
- {
- HeaderTag *tmpl_tag = tmpl_tags->data;
- HeaderTag *out_tag = header_line_has_tag(out_hline, tmpl_tag->key);
- if ( !out_tag )
- {
- HeaderTag *tag = malloc(sizeof(HeaderTag));
- tag->key[0] = tmpl_tag->key[0];
- tag->key[1] = tmpl_tag->key[1];
- tag->value = strdup(tmpl_tag->value);
- out_hline->tags = list_append(out_hline->tags,tag);
- }
- tmpl_tags = tmpl_tags->next;
- }
- return 1;
-}
-
-
-static HeaderLine *sam_header_line_parse(const char *headerLine)
-{
- HeaderLine *hline;
- HeaderTag *tag;
- const char *from, *to;
- from = headerLine;
-
- if ( *from != '@' ) {
- debug("[sam_header_line_parse] expected '@', got [%s]\n", headerLine);
- return 0;
- }
- to = ++from;
-
- while (*to && *to!='\t') to++;
- if ( to-from != 2 ) {
- debug("[sam_header_line_parse] expected '@XY', got [%s]\nHint: The header tags must be tab-separated.\n", headerLine);
- return 0;
- }
-
- hline = malloc(sizeof(HeaderLine));
- hline->type[0] = from[0];
- hline->type[1] = from[1];
- hline->tags = NULL;
-
- int itype = tag_exists(hline->type, types);
-
- from = to;
- while (*to && *to=='\t') to++;
- if ( to-from != 1 ) {
- debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from));
- free(hline);
- return 0;
- }
- from = to;
- while (*from)
- {
- while (*to && *to!='\t') to++;
-
- if ( !required_tags[itype] && !optional_tags[itype] )
- {
- // CO is a special case, it can contain anything, including tabs
- if ( *to ) { to++; continue; }
- tag = new_tag(" ",from,to-1);
- }
- else
- tag = new_tag(from,from+3,to-1);
-
- if ( header_line_has_tag(hline,tag->key) )
- debug("The tag '%c%c' present (at least) twice on line [%s]\n", tag->key[0],tag->key[1], headerLine);
- hline->tags = list_append(hline->tags, tag);
-
- from = to;
- while (*to && *to=='\t') to++;
- if ( *to && to-from != 1 ) {
- debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from));
- return 0;
- }
-
- from = to;
- }
- return hline;
-}
-
-
-// Must be of an existing type, all tags must be recognised and all required tags must be present
-static int sam_header_line_validate(HeaderLine *hline)
-{
- list_t *tags;
- HeaderTag *tag;
- int itype, itag;
-
- // Is the type correct?
- itype = tag_exists(hline->type, types);
- if ( itype==-1 )
- {
- debug("The type [%c%c] not recognised.\n", hline->type[0],hline->type[1]);
- return 0;
- }
-
- // Has all required tags?
- itag = 0;
- while ( required_tags[itype] && required_tags[itype][itag] )
- {
- if ( !header_line_has_tag(hline,required_tags[itype][itag]) )
- {
- debug("The tag [%c%c] required for [%c%c] not present.\n", required_tags[itype][itag][0],required_tags[itype][itag][1],
- hline->type[0],hline->type[1]);
- return 0;
- }
- itag++;
- }
-
- // Are all tags recognised?
- tags = hline->tags;
- while ( tags )
- {
- tag = tags->data;
- if ( !tag_exists(tag->key,required_tags[itype]) && !tag_exists(tag->key,optional_tags[itype]) )
- {
- // Lower case tags are user-defined values.
- if( !(islower(tag->key[0]) || islower(tag->key[1])) )
- {
- // Neither is lower case, but tag was not recognized.
- debug("Unknown tag [%c%c] for [%c%c].\n", tag->key[0],tag->key[1], hline->type[0],hline->type[1]);
- // return 0; // Even unknown tags are allowed - for forward compatibility with new attributes
- }
- // else - allow user defined tag
- }
- tags = tags->next;
- }
-
- return 1;
-}
-
-
-static void print_header_line(FILE *fp, HeaderLine *hline)
-{
- list_t *tags = hline->tags;
- HeaderTag *tag;
-
- fprintf(fp, "@%c%c", hline->type[0],hline->type[1]);
- while (tags)
- {
- tag = tags->data;
-
- fprintf(fp, "\t");
- if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
- fprintf(fp, "%c%c:", tag->key[0],tag->key[1]);
- fprintf(fp, "%s", tag->value);
-
- tags = tags->next;
- }
- fprintf(fp,"\n");
-}
-
-
-static void sam_header_line_free(HeaderLine *hline)
-{
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- free(tag->value);
- free(tag);
- tags = tags->next;
- }
- list_free(hline->tags);
- free(hline);
-}
-
-void sam_header_free(void *_header)
-{
- HeaderDict *header = (HeaderDict*)_header;
- list_t *hlines = header;
- while (hlines)
- {
- sam_header_line_free(hlines->data);
- hlines = hlines->next;
- }
- list_free(header);
-}
-
-HeaderDict *sam_header_clone(const HeaderDict *dict)
-{
- HeaderDict *out = NULL;
- while (dict)
- {
- HeaderLine *hline = dict->data;
- out = list_append(out, sam_header_line_clone(hline));
- dict = dict->next;
- }
- return out;
-}
-
-// Returns a newly allocated string
-char *sam_header_write(const void *_header)
-{
- const HeaderDict *header = (const HeaderDict*)_header;
- char *out = NULL;
- int len=0, nout=0;
- const list_t *hlines;
-
- // Calculate the length of the string to allocate
- hlines = header;
- while (hlines)
- {
- len += 4; // @XY and \n
-
- HeaderLine *hline = hlines->data;
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- len += strlen(tag->value) + 1; // \t
- if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
- len += strlen(tag->value) + 3; // XY:
- tags = tags->next;
- }
- hlines = hlines->next;
- }
-
- nout = 0;
- out = malloc(len+1);
- hlines = header;
- while (hlines)
- {
- HeaderLine *hline = hlines->data;
-
- nout += sprintf(out+nout,"@%c%c",hline->type[0],hline->type[1]);
-
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- nout += sprintf(out+nout,"\t");
- if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
- nout += sprintf(out+nout,"%c%c:", tag->key[0],tag->key[1]);
- nout += sprintf(out+nout,"%s", tag->value);
- tags = tags->next;
- }
- hlines = hlines->next;
- nout += sprintf(out+nout,"\n");
- }
- out[len] = 0;
- return out;
-}
-
-void *sam_header_parse2(const char *headerText)
-{
- list_t *hlines = NULL;
- HeaderLine *hline;
- const char *text;
- char *buf=NULL;
- size_t nbuf = 0;
- int tovalidate = 0;
-
- if ( !headerText )
- return 0;
-
- text = headerText;
- while ( (text=nextline(&buf, &nbuf, text)) )
- {
- hline = sam_header_line_parse(buf);
- if ( hline && (!tovalidate || sam_header_line_validate(hline)) )
- // With too many (~250,000) reference sequences the header parsing was too slow with list_append.
- hlines = list_append_to_end(hlines, hline);
- else
- {
- if (hline) sam_header_line_free(hline);
- sam_header_free(hlines);
- if ( buf ) free(buf);
- return NULL;
- }
- }
- if ( buf ) free(buf);
-
- return hlines;
-}
-
-void *sam_header2tbl(const void *_dict, char type[2], char key_tag[2], char value_tag[2])
-{
- const HeaderDict *dict = (const HeaderDict*)_dict;
- const list_t *l = dict;
- khash_t(str) *tbl = kh_init(str);
- khiter_t k;
- int ret;
-
- if (_dict == 0) return tbl; // return an empty (not null) hash table
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
-
- HeaderTag *key, *value;
- key = header_line_has_tag(hline,key_tag);
- value = header_line_has_tag(hline,value_tag);
- if ( !key || !value )
- {
- l = l->next;
- continue;
- }
-
- k = kh_get(str, tbl, key->value);
- if ( k != kh_end(tbl) )
- debug("[sam_header_lookup_table] They key %s not unique.\n", key->value);
- k = kh_put(str, tbl, key->value, &ret);
- kh_value(tbl, k) = value->value;
-
- l = l->next;
- }
- return tbl;
-}
-
-char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n)
-{
- const HeaderDict *dict = (const HeaderDict*)_dict;
- const list_t *l = dict;
- int max, n;
- char **ret;
-
- ret = 0; *_n = max = n = 0;
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
-
- HeaderTag *key;
- key = header_line_has_tag(hline,key_tag);
- if ( !key )
- {
- l = l->next;
- continue;
- }
-
- if (n == max) {
- max = max? max<<1 : 4;
- ret = realloc(ret, max * sizeof(char*));
- }
- ret[n++] = key->value;
-
- l = l->next;
- }
- *_n = n;
- return ret;
-}
-
-void *sam_header2key_val(void *iter, const char type[2], const char key_tag[2], const char value_tag[2], const char **_key, const char **_value)
-{
- list_t *l = iter;
- if ( !l ) return NULL;
-
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
-
- HeaderTag *key, *value;
- key = header_line_has_tag(hline,key_tag);
- value = header_line_has_tag(hline,value_tag);
- if ( !key || !value )
- {
- l = l->next;
- continue;
- }
-
- *_key = key->value;
- *_value = value->value;
- return l->next;
- }
- return l;
-}
-
-const char *sam_tbl_get(void *h, const char *key)
-{
- khash_t(str) *tbl = (khash_t(str)*)h;
- khint_t k;
- k = kh_get(str, tbl, key);
- return k == kh_end(tbl)? 0 : kh_val(tbl, k);
-}
-
-int sam_tbl_size(void *h)
-{
- khash_t(str) *tbl = (khash_t(str)*)h;
- return h? kh_size(tbl) : 0;
-}
-
-void sam_tbl_destroy(void *h)
-{
- khash_t(str) *tbl = (khash_t(str)*)h;
- kh_destroy(str, tbl);
-}
-
-void *sam_header_merge(int n, const void **_dicts)
-{
- const HeaderDict **dicts = (const HeaderDict**)_dicts;
- HeaderDict *out_dict;
- int idict, status;
-
- if ( n<2 ) return NULL;
-
- out_dict = sam_header_clone(dicts[0]);
-
- for (idict=1; idict<n; idict++)
- {
- const list_t *tmpl_hlines = dicts[idict];
-
- while ( tmpl_hlines )
- {
- list_t *out_hlines = out_dict;
- int inserted = 0;
- while ( out_hlines )
- {
- status = sam_header_compare_lines(tmpl_hlines->data, out_hlines->data);
- if ( status==0 )
- {
- out_hlines = out_hlines->next;
- continue;
- }
-
- if ( status==2 )
- {
- print_header_line(stderr,tmpl_hlines->data);
- print_header_line(stderr,out_hlines->data);
- debug("Conflicting lines, cannot merge the headers.\n");
- return 0;
- }
- if ( status==3 )
- sam_header_line_merge_with(out_hlines->data, tmpl_hlines->data);
-
- inserted = 1;
- break;
- }
- if ( !inserted )
- out_dict = list_append(out_dict, sam_header_line_clone(tmpl_hlines->data));
-
- tmpl_hlines = tmpl_hlines->next;
- }
- }
-
- return out_dict;
-}
-
-char **sam_header2tbl_n(const void *dict, const char type[2], const char *tags[], int *n)
-{
- int nout = 0;
- char **out = NULL;
-
- *n = 0;
- list_t *l = (list_t *)dict;
- if ( !l ) return NULL;
-
- int i, ntags = 0;
- while ( tags[ntags] ) ntags++;
-
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
- out = (char**) realloc(out, sizeof(char*)*(nout+1)*ntags);
- for (i=0; i<ntags; i++)
- {
- HeaderTag *key = header_line_has_tag(hline, tags[i]);
- if ( !key )
- {
- out[nout*ntags+i] = NULL;
- continue;
- }
- out[nout*ntags+i] = key->value;
- }
- nout++;
- l = l->next;
- }
- *n = nout;
- return out;
-}
-
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* sam_header.c -- basic SAM/BAM header API.
-
- Copyright (C) 2009-2013 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "sam_header.h"
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
-#include <stdlib.h>
-#include <stdarg.h>
-
-#include "htslib/khash.h"
-KHASH_MAP_INIT_STR(str, const char *)
-
-struct _HeaderList
-{
- struct _HeaderList *last; // Hack: Used and maintained only by list_append_to_end. Maintained in the root node only.
- struct _HeaderList *next;
- void *data;
-};
-typedef struct _HeaderList list_t;
-typedef list_t HeaderDict;
-
-typedef struct
-{
- char key[2];
- char *value;
-}
-HeaderTag;
-
-typedef struct
-{
- char type[2];
- list_t *tags;
-}
-HeaderLine;
-
-const char *o_hd_tags[] = {"SO","GO",NULL};
-const char *r_hd_tags[] = {"VN",NULL};
-
-const char *o_sq_tags[] = {"AS","M5","UR","SP",NULL};
-const char *r_sq_tags[] = {"SN","LN",NULL};
-const char *u_sq_tags[] = {"SN",NULL};
-
-const char *o_rg_tags[] = {"CN","DS","DT","FO","KS","LB","PG","PI","PL","PU","SM",NULL};
-const char *r_rg_tags[] = {"ID",NULL};
-const char *u_rg_tags[] = {"ID",NULL};
-
-const char *o_pg_tags[] = {"VN","CL",NULL};
-const char *r_pg_tags[] = {"ID",NULL};
-
-const char *types[] = {"HD","SQ","RG","PG","CO",NULL};
-const char **optional_tags[] = {o_hd_tags,o_sq_tags,o_rg_tags,o_pg_tags,NULL,NULL};
-const char **required_tags[] = {r_hd_tags,r_sq_tags,r_rg_tags,r_pg_tags,NULL,NULL};
-const char **unique_tags[] = {NULL, u_sq_tags,u_rg_tags,NULL,NULL,NULL};
-
-
-static void debug(const char *format, ...)
-{
- va_list ap;
- va_start(ap, format);
- vfprintf(samtools_stderr, format, ap);
- va_end(ap);
-}
-
-#if 0
-// Replaced by list_append_to_end
-static list_t *list_prepend(list_t *root, void *data)
-{
- list_t *l = malloc(sizeof(list_t));
- l->next = root;
- l->data = data;
- return l;
-}
-#endif
-
-// Relies on the root->last being correct. Do not use with the other list_*
-// routines unless they are fixed to modify root->last as well.
-static list_t *list_append_to_end(list_t *root, void *data)
-{
- list_t *l = malloc(sizeof(list_t));
- l->last = l;
- l->next = NULL;
- l->data = data;
-
- if ( !root )
- return l;
-
- root->last->next = l;
- root->last = l;
- return root;
-}
-
-static list_t *list_append(list_t *root, void *data)
-{
- list_t *l = root;
- while (l && l->next)
- l = l->next;
- if ( l )
- {
- l->next = malloc(sizeof(list_t));
- l = l->next;
- }
- else
- {
- l = malloc(sizeof(list_t));
- root = l;
- }
- l->data = data;
- l->next = NULL;
- return root;
-}
-
-static void list_free(list_t *root)
-{
- list_t *l = root;
- while (root)
- {
- l = root;
- root = root->next;
- free(l);
- }
-}
-
-
-
-// Look for a tag "XY" in a predefined const char *[] array.
-static int tag_exists(const char *tag, const char **tags)
-{
- int itag=0;
- if ( !tags ) return -1;
- while ( tags[itag] )
- {
- if ( tags[itag][0]==tag[0] && tags[itag][1]==tag[1] ) return itag;
- itag++;
- }
- return -1;
-}
-
-
-
-// Mimics the behaviour of getline, except it returns pointer to the next chunk of the text
-// or NULL if everything has been read. The lineptr should be freed by the caller. The
-// newline character is stripped.
-static const char *nextline(char **lineptr, size_t *n, const char *text)
-{
- int len;
- const char *to = text;
-
- if ( !*to ) return NULL;
-
- while ( *to && *to!='\n' && *to!='\r' ) to++;
- len = to - text + 1;
-
- if ( *to )
- {
- // Advance the pointer for the next call
- if ( *to=='\n' ) to++;
- else if ( *to=='\r' && *(to+1)=='\n' ) to+=2;
- }
- if ( !len )
- return to;
-
- if ( !*lineptr )
- {
- *lineptr = malloc(len);
- *n = len;
- }
- else if ( *n<len )
- {
- *lineptr = realloc(*lineptr, len);
- *n = len;
- }
- if ( !*lineptr ) {
- debug("[nextline] Insufficient memory!\n");
- return 0;
- }
-
- memcpy(*lineptr,text,len);
- (*lineptr)[len-1] = 0;
-
- return to;
-}
-
-// name points to "XY", value_from points to the first character of the value string and
-// value_to points to the last character of the value string.
-static HeaderTag *new_tag(const char *name, const char *value_from, const char *value_to)
-{
- HeaderTag *tag = malloc(sizeof(HeaderTag));
- int len = value_to-value_from+1;
-
- tag->key[0] = name[0];
- tag->key[1] = name[1];
- tag->value = malloc(len+1);
- memcpy(tag->value,value_from,len+1);
- tag->value[len] = 0;
- return tag;
-}
-
-static HeaderTag *header_line_has_tag(HeaderLine *hline, const char *key)
-{
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- if ( tag->key[0]==key[0] && tag->key[1]==key[1] ) return tag;
- tags = tags->next;
- }
- return NULL;
-}
-
-
-// Return codes:
-// 0 .. different types or unique tags differ or conflicting tags, cannot be merged
-// 1 .. all tags identical -> no need to merge, drop one
-// 2 .. the unique tags match and there are some conflicting tags (same tag, different value) -> error, cannot be merged nor duplicated
-// 3 .. there are some missing complementary tags and no unique conflict -> can be merged into a single line
-static int sam_header_compare_lines(HeaderLine *hline1, HeaderLine *hline2)
-{
- HeaderTag *t1, *t2;
-
- if ( hline1->type[0]!=hline2->type[0] || hline1->type[1]!=hline2->type[1] )
- return 0;
-
- int itype = tag_exists(hline1->type,types);
- if ( itype==-1 ) {
- debug("[sam_header_compare_lines] Unknown type [%c%c]\n", hline1->type[0],hline1->type[1]);
- return -1; // FIXME (lh3): error; I do not know how this will be handled in Petr's code
- }
-
- if ( unique_tags[itype] )
- {
- t1 = header_line_has_tag(hline1,unique_tags[itype][0]);
- t2 = header_line_has_tag(hline2,unique_tags[itype][0]);
- if ( !t1 || !t2 ) // this should never happen, the unique tags are required
- return 2;
-
- if ( strcmp(t1->value,t2->value) )
- return 0; // the unique tags differ, cannot be merged
- }
- if ( !required_tags[itype] && !optional_tags[itype] )
- {
- t1 = hline1->tags->data;
- t2 = hline2->tags->data;
- if ( !strcmp(t1->value,t2->value) ) return 1; // identical comments
- return 0;
- }
-
- int missing=0, itag=0;
- while ( required_tags[itype] && required_tags[itype][itag] )
- {
- t1 = header_line_has_tag(hline1,required_tags[itype][itag]);
- t2 = header_line_has_tag(hline2,required_tags[itype][itag]);
- if ( !t1 && !t2 )
- return 2; // this should never happen
- else if ( !t1 || !t2 )
- missing = 1; // there is some tag missing in one of the hlines
- else if ( strcmp(t1->value,t2->value) )
- {
- if ( unique_tags[itype] )
- return 2; // the lines have a matching unique tag but have a conflicting tag
-
- return 0; // the lines contain conflicting tags, cannot be merged
- }
- itag++;
- }
- itag = 0;
- while ( optional_tags[itype] && optional_tags[itype][itag] )
- {
- t1 = header_line_has_tag(hline1,optional_tags[itype][itag]);
- t2 = header_line_has_tag(hline2,optional_tags[itype][itag]);
- if ( !t1 && !t2 )
- {
- itag++;
- continue;
- }
- if ( !t1 || !t2 )
- missing = 1; // there is some tag missing in one of the hlines
- else if ( strcmp(t1->value,t2->value) )
- {
- if ( unique_tags[itype] )
- return 2; // the lines have a matching unique tag but have a conflicting tag
-
- return 0; // the lines contain conflicting tags, cannot be merged
- }
- itag++;
- }
- if ( missing ) return 3; // there are some missing complementary tags with no conflicts, can be merged
- return 1;
-}
-
-
-static HeaderLine *sam_header_line_clone(const HeaderLine *hline)
-{
- list_t *tags;
- HeaderLine *out = malloc(sizeof(HeaderLine));
- out->type[0] = hline->type[0];
- out->type[1] = hline->type[1];
- out->tags = NULL;
-
- tags = hline->tags;
- while (tags)
- {
- HeaderTag *old = tags->data;
-
- HeaderTag *new = malloc(sizeof(HeaderTag));
- new->key[0] = old->key[0];
- new->key[1] = old->key[1];
- new->value = strdup(old->value);
- out->tags = list_append(out->tags, new);
-
- tags = tags->next;
- }
- return out;
-}
-
-static int sam_header_line_merge_with(HeaderLine *out_hline, const HeaderLine *tmpl_hline)
-{
- list_t *tmpl_tags;
-
- if ( out_hline->type[0]!=tmpl_hline->type[0] || out_hline->type[1]!=tmpl_hline->type[1] )
- return 0;
-
- tmpl_tags = tmpl_hline->tags;
- while (tmpl_tags)
- {
- HeaderTag *tmpl_tag = tmpl_tags->data;
- HeaderTag *out_tag = header_line_has_tag(out_hline, tmpl_tag->key);
- if ( !out_tag )
- {
- HeaderTag *tag = malloc(sizeof(HeaderTag));
- tag->key[0] = tmpl_tag->key[0];
- tag->key[1] = tmpl_tag->key[1];
- tag->value = strdup(tmpl_tag->value);
- out_hline->tags = list_append(out_hline->tags,tag);
- }
- tmpl_tags = tmpl_tags->next;
- }
- return 1;
-}
-
-
-static HeaderLine *sam_header_line_parse(const char *headerLine)
-{
- HeaderLine *hline;
- HeaderTag *tag;
- const char *from, *to;
- from = headerLine;
-
- if ( *from != '@' ) {
- debug("[sam_header_line_parse] expected '@', got [%s]\n", headerLine);
- return 0;
- }
- to = ++from;
-
- while (*to && *to!='\t') to++;
- if ( to-from != 2 ) {
- debug("[sam_header_line_parse] expected '@XY', got [%s]\nHint: The header tags must be tab-separated.\n", headerLine);
- return 0;
- }
-
- hline = malloc(sizeof(HeaderLine));
- hline->type[0] = from[0];
- hline->type[1] = from[1];
- hline->tags = NULL;
-
- int itype = tag_exists(hline->type, types);
-
- from = to;
- while (*to && *to=='\t') to++;
- if ( to-from != 1 ) {
- debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from));
- free(hline);
- return 0;
- }
- from = to;
- while (*from)
- {
- while (*to && *to!='\t') to++;
-
- if ( !required_tags[itype] && !optional_tags[itype] )
- {
- // CO is a special case, it can contain anything, including tabs
- if ( *to ) { to++; continue; }
- tag = new_tag(" ",from,to-1);
- }
- else
- tag = new_tag(from,from+3,to-1);
-
- if ( header_line_has_tag(hline,tag->key) )
- debug("The tag '%c%c' present (at least) twice on line [%s]\n", tag->key[0],tag->key[1], headerLine);
- hline->tags = list_append(hline->tags, tag);
-
- from = to;
- while (*to && *to=='\t') to++;
- if ( *to && to-from != 1 ) {
- debug("[sam_header_line_parse] multiple tabs on line [%s] (%d)\n", headerLine,(int)(to-from));
- return 0;
- }
-
- from = to;
- }
- return hline;
-}
-
-
-// Must be of an existing type, all tags must be recognised and all required tags must be present
-static int sam_header_line_validate(HeaderLine *hline)
-{
- list_t *tags;
- HeaderTag *tag;
- int itype, itag;
-
- // Is the type correct?
- itype = tag_exists(hline->type, types);
- if ( itype==-1 )
- {
- debug("The type [%c%c] not recognised.\n", hline->type[0],hline->type[1]);
- return 0;
- }
-
- // Has all required tags?
- itag = 0;
- while ( required_tags[itype] && required_tags[itype][itag] )
- {
- if ( !header_line_has_tag(hline,required_tags[itype][itag]) )
- {
- debug("The tag [%c%c] required for [%c%c] not present.\n", required_tags[itype][itag][0],required_tags[itype][itag][1],
- hline->type[0],hline->type[1]);
- return 0;
- }
- itag++;
- }
-
- // Are all tags recognised?
- tags = hline->tags;
- while ( tags )
- {
- tag = tags->data;
- if ( !tag_exists(tag->key,required_tags[itype]) && !tag_exists(tag->key,optional_tags[itype]) )
- {
- // Lower case tags are user-defined values.
- if( !(islower(tag->key[0]) || islower(tag->key[1])) )
- {
- // Neither is lower case, but tag was not recognized.
- debug("Unknown tag [%c%c] for [%c%c].\n", tag->key[0],tag->key[1], hline->type[0],hline->type[1]);
- // return 0; // Even unknown tags are allowed - for forward compatibility with new attributes
- }
- // else - allow user defined tag
- }
- tags = tags->next;
- }
-
- return 1;
-}
-
-
-static void print_header_line(FILE *fp, HeaderLine *hline)
-{
- list_t *tags = hline->tags;
- HeaderTag *tag;
-
- fprintf(fp, "@%c%c", hline->type[0],hline->type[1]);
- while (tags)
- {
- tag = tags->data;
-
- fprintf(fp, "\t");
- if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
- fprintf(fp, "%c%c:", tag->key[0],tag->key[1]);
- fprintf(fp, "%s", tag->value);
-
- tags = tags->next;
- }
- fprintf(fp,"\n");
-}
-
-
-static void sam_header_line_free(HeaderLine *hline)
-{
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- free(tag->value);
- free(tag);
- tags = tags->next;
- }
- list_free(hline->tags);
- free(hline);
-}
-
-void sam_header_free(void *_header)
-{
- HeaderDict *header = (HeaderDict*)_header;
- list_t *hlines = header;
- while (hlines)
- {
- sam_header_line_free(hlines->data);
- hlines = hlines->next;
- }
- list_free(header);
-}
-
-HeaderDict *sam_header_clone(const HeaderDict *dict)
-{
- HeaderDict *out = NULL;
- while (dict)
- {
- HeaderLine *hline = dict->data;
- out = list_append(out, sam_header_line_clone(hline));
- dict = dict->next;
- }
- return out;
-}
-
-// Returns a newly allocated string
-char *sam_header_write(const void *_header)
-{
- const HeaderDict *header = (const HeaderDict*)_header;
- char *out = NULL;
- int len=0, nout=0;
- const list_t *hlines;
-
- // Calculate the length of the string to allocate
- hlines = header;
- while (hlines)
- {
- len += 4; // @XY and \n
-
- HeaderLine *hline = hlines->data;
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- len += strlen(tag->value) + 1; // \t
- if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
- len += strlen(tag->value) + 3; // XY:
- tags = tags->next;
- }
- hlines = hlines->next;
- }
-
- nout = 0;
- out = malloc(len+1);
- hlines = header;
- while (hlines)
- {
- HeaderLine *hline = hlines->data;
-
- nout += sprintf(out+nout,"@%c%c",hline->type[0],hline->type[1]);
-
- list_t *tags = hline->tags;
- while (tags)
- {
- HeaderTag *tag = tags->data;
- nout += sprintf(out+nout,"\t");
- if ( tag->key[0]!=' ' || tag->key[1]!=' ' )
- nout += sprintf(out+nout,"%c%c:", tag->key[0],tag->key[1]);
- nout += sprintf(out+nout,"%s", tag->value);
- tags = tags->next;
- }
- hlines = hlines->next;
- nout += sprintf(out+nout,"\n");
- }
- out[len] = 0;
- return out;
-}
-
-void *sam_header_parse2(const char *headerText)
-{
- list_t *hlines = NULL;
- HeaderLine *hline;
- const char *text;
- char *buf=NULL;
- size_t nbuf = 0;
- int tovalidate = 0;
-
- if ( !headerText )
- return 0;
-
- text = headerText;
- while ( (text=nextline(&buf, &nbuf, text)) )
- {
- hline = sam_header_line_parse(buf);
- if ( hline && (!tovalidate || sam_header_line_validate(hline)) )
- // With too many (~250,000) reference sequences the header parsing was too slow with list_append.
- hlines = list_append_to_end(hlines, hline);
- else
- {
- if (hline) sam_header_line_free(hline);
- sam_header_free(hlines);
- if ( buf ) free(buf);
- return NULL;
- }
- }
- if ( buf ) free(buf);
-
- return hlines;
-}
-
-void *sam_header2tbl(const void *_dict, char type[2], char key_tag[2], char value_tag[2])
-{
- const HeaderDict *dict = (const HeaderDict*)_dict;
- const list_t *l = dict;
- khash_t(str) *tbl = kh_init(str);
- khiter_t k;
- int ret;
-
- if (_dict == 0) return tbl; // return an empty (not null) hash table
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
-
- HeaderTag *key, *value;
- key = header_line_has_tag(hline,key_tag);
- value = header_line_has_tag(hline,value_tag);
- if ( !key || !value )
- {
- l = l->next;
- continue;
- }
-
- k = kh_get(str, tbl, key->value);
- if ( k != kh_end(tbl) )
- debug("[sam_header_lookup_table] They key %s not unique.\n", key->value);
- k = kh_put(str, tbl, key->value, &ret);
- kh_value(tbl, k) = value->value;
-
- l = l->next;
- }
- return tbl;
-}
-
-char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n)
-{
- const HeaderDict *dict = (const HeaderDict*)_dict;
- const list_t *l = dict;
- int max, n;
- char **ret;
-
- ret = 0; *_n = max = n = 0;
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
-
- HeaderTag *key;
- key = header_line_has_tag(hline,key_tag);
- if ( !key )
- {
- l = l->next;
- continue;
- }
-
- if (n == max) {
- max = max? max<<1 : 4;
- ret = realloc(ret, max * sizeof(char*));
- }
- ret[n++] = key->value;
-
- l = l->next;
- }
- *_n = n;
- return ret;
-}
-
-void *sam_header2key_val(void *iter, const char type[2], const char key_tag[2], const char value_tag[2], const char **_key, const char **_value)
-{
- list_t *l = iter;
- if ( !l ) return NULL;
-
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
-
- HeaderTag *key, *value;
- key = header_line_has_tag(hline,key_tag);
- value = header_line_has_tag(hline,value_tag);
- if ( !key || !value )
- {
- l = l->next;
- continue;
- }
-
- *_key = key->value;
- *_value = value->value;
- return l->next;
- }
- return l;
-}
-
-const char *sam_tbl_get(void *h, const char *key)
-{
- khash_t(str) *tbl = (khash_t(str)*)h;
- khint_t k;
- k = kh_get(str, tbl, key);
- return k == kh_end(tbl)? 0 : kh_val(tbl, k);
-}
-
-int sam_tbl_size(void *h)
-{
- khash_t(str) *tbl = (khash_t(str)*)h;
- return h? kh_size(tbl) : 0;
-}
-
-void sam_tbl_destroy(void *h)
-{
- khash_t(str) *tbl = (khash_t(str)*)h;
- kh_destroy(str, tbl);
-}
-
-void *sam_header_merge(int n, const void **_dicts)
-{
- const HeaderDict **dicts = (const HeaderDict**)_dicts;
- HeaderDict *out_dict;
- int idict, status;
-
- if ( n<2 ) return NULL;
-
- out_dict = sam_header_clone(dicts[0]);
-
- for (idict=1; idict<n; idict++)
- {
- const list_t *tmpl_hlines = dicts[idict];
-
- while ( tmpl_hlines )
- {
- list_t *out_hlines = out_dict;
- int inserted = 0;
- while ( out_hlines )
- {
- status = sam_header_compare_lines(tmpl_hlines->data, out_hlines->data);
- if ( status==0 )
- {
- out_hlines = out_hlines->next;
- continue;
- }
-
- if ( status==2 )
- {
- print_header_line(samtools_stderr,tmpl_hlines->data);
- print_header_line(samtools_stderr,out_hlines->data);
- debug("Conflicting lines, cannot merge the headers.\n");
- return 0;
- }
- if ( status==3 )
- sam_header_line_merge_with(out_hlines->data, tmpl_hlines->data);
-
- inserted = 1;
- break;
- }
- if ( !inserted )
- out_dict = list_append(out_dict, sam_header_line_clone(tmpl_hlines->data));
-
- tmpl_hlines = tmpl_hlines->next;
- }
- }
-
- return out_dict;
-}
-
-char **sam_header2tbl_n(const void *dict, const char type[2], const char *tags[], int *n)
-{
- int nout = 0;
- char **out = NULL;
-
- *n = 0;
- list_t *l = (list_t *)dict;
- if ( !l ) return NULL;
-
- int i, ntags = 0;
- while ( tags[ntags] ) ntags++;
-
- while (l)
- {
- HeaderLine *hline = l->data;
- if ( hline->type[0]!=type[0] || hline->type[1]!=type[1] )
- {
- l = l->next;
- continue;
- }
- out = (char**) realloc(out, sizeof(char*)*(nout+1)*ntags);
- for (i=0; i<ntags; i++)
- {
- HeaderTag *key = header_line_has_tag(hline, tags[i]);
- if ( !key )
- {
- out[nout*ntags+i] = NULL;
- continue;
- }
- out[nout*ntags+i] = key->value;
- }
- nout++;
- l = l->next;
- }
- *n = nout;
- return out;
-}
-
+++ /dev/null
-/* sam_header.h -- basic SAM/BAM header API.
-
- Copyright (C) 2009, 2012, 2013 Genome Research Ltd.
-
- Author: Petr Danecek <pd3@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#ifndef __SAM_HEADER_H__
-#define __SAM_HEADER_H__
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- void *sam_header_parse2(const char *headerText);
- void *sam_header_merge(int n, const void **dicts);
- void sam_header_free(void *header);
- char *sam_header_write(const void *headerDict); // returns a newly allocated string
-
- /*
- // Usage example
- const char *key, *val;
- void *iter = sam_header_parse2(bam->header->text);
- while ( iter = sam_header_key_val(iter, "RG","ID","SM" &key,&val) ) printf("%s\t%s\n", key,val);
- */
- void *sam_header2key_val(void *iter, const char type[2], const char key_tag[2], const char value_tag[2], const char **key, const char **value);
- char **sam_header2list(const void *_dict, char type[2], char key_tag[2], int *_n);
-
- /*
- // Usage example
- int i, j, n;
- const char *tags[] = {"SN","LN","UR","M5",NULL};
- void *dict = sam_header_parse2(bam->header->text);
- char **tbl = sam_header2tbl_n(h->dict, "SQ", tags, &n);
- for (i=0; i<n; i++)
- {
- for (j=0; j<4; j++)
- if ( tbl[4*i+j] ) printf("\t%s", tbl[4*i+j]);
- else printf("-");
- printf("\n");
- }
- if (tbl) free(tbl);
- */
- char **sam_header2tbl_n(const void *dict, const char type[2], const char *tags[], int *n);
-
- void *sam_header2tbl(const void *dict, char type[2], char key_tag[2], char value_tag[2]);
- const char *sam_tbl_get(void *h, const char *key);
- int sam_tbl_size(void *h);
- void sam_tbl_destroy(void *h);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
/* sam_opts.c -- utilities to aid parsing common command line options.
- Copyright (C) 2015 Genome Research Ltd.
+ Copyright (C) 2015, 2019 Genome Research Ltd.
Author: James Bonfield <jkb@sanger.ac.uk>
break;
} else if (strcmp(lopt->name, "reference") == 0) {
char *ref = malloc(10 + strlen(optarg) + 1);
+
+ if (!ref) {
+ fprintf(stderr, "Unable to allocate memory in "
+ "parse_sam_global_opt.\n");
+
+ return -1;
+ }
+
sprintf(ref, "reference=%s", optarg);
- ga->reference = strdup(optarg);
+
+ if (!(ga->reference = strdup(optarg))) {
+ fprintf(stderr, "Unable to allocate memory in "
+ "parse_sam_global_opt.\n");
+
+ return -1;
+ }
+
r = hts_opt_add((hts_opt **)&ga->in.specific, ref);
r |= hts_opt_add((hts_opt **)&ga->out.specific, ref);
free(ref);
} else if (strcmp(lopt->name, "threads") == 0) {
ga->nthreads = atoi(optarg);
break;
-// } else if (strcmp(lopt->name, "verbose") == 0) {
-// ga->verbosity++;
-// break;
+ } else if (strcmp(lopt->name, "write-index") == 0) {
+ ga->write_index = 1;
+ break;
+ } else if (strcmp(lopt->name, "verbosity") == 0) {
+ hts_verbose = atoi(optarg);
+ break;
}
}
if (!lopt->name) {
- fprintf(stderr, "Unexpected global option: %s\n", lopt->name);
+ fprintf(stderr, "Unexpected global option.\n");
return -1;
}
+ /*
+ * SAM format with compression enabled implies SAM.bgzf
+ */
+ if (ga->out.format == sam) {
+ hts_opt *opts = (hts_opt *)ga->out.specific;
+ while (opts) {
+ if (opts->opt == HTS_OPT_COMPRESSION_LEVEL)
+ ga->out.compression = bgzf;
+ opts = opts->next;
+ }
+ }
+
return r;
}
else if (strcmp(lopts[i].name, "threads") == 0)
fprintf(fp,"threads INT\n"
" Number of additional threads to use [0]\n");
-// else if (strcmp(lopts[i].name, "verbose") == 0)
-// fprintf(fp,"verbose\n"
-// " Increment level of verbosity\n");
+ else if (strcmp(lopts[i].name, "write-index") == 0)
+ fprintf(fp,"write-index\n"
+ " Automatically index the output files [off]\n");
+ else if (strcmp(lopts[i].name, "verbosity") == 0)
+ fprintf(fp,"verbosity INT\n"
+ " Set level of verbosity\n");
}
}
/* sam_opts.c -- utilities to aid parsing common command line options.
- Copyright (C) 2015 Genome Research Ltd.
+ Copyright (C) 2015, 2019 Genome Research Ltd.
Author: James Bonfield <jkb@sanger.ac.uk>
break;
} else if (strcmp(lopt->name, "reference") == 0) {
char *ref = malloc(10 + strlen(optarg) + 1);
+
+ if (!ref) {
+ fprintf(samtools_stderr, "Unable to allocate memory in "
+ "parse_sam_global_opt.\n");
+
+ return -1;
+ }
+
sprintf(ref, "reference=%s", optarg);
- ga->reference = strdup(optarg);
+
+ if (!(ga->reference = strdup(optarg))) {
+ fprintf(samtools_stderr, "Unable to allocate memory in "
+ "parse_sam_global_opt.\n");
+
+ return -1;
+ }
+
r = hts_opt_add((hts_opt **)&ga->in.specific, ref);
r |= hts_opt_add((hts_opt **)&ga->out.specific, ref);
free(ref);
} else if (strcmp(lopt->name, "threads") == 0) {
ga->nthreads = atoi(optarg);
break;
-// } else if (strcmp(lopt->name, "verbose") == 0) {
-// ga->verbosity++;
-// break;
+ } else if (strcmp(lopt->name, "write-index") == 0) {
+ ga->write_index = 1;
+ break;
+ } else if (strcmp(lopt->name, "verbosity") == 0) {
+ hts_verbose = atoi(optarg);
+ break;
}
}
if (!lopt->name) {
- fprintf(samtools_stderr, "Unexpected global option: %s\n", lopt->name);
+ fprintf(samtools_stderr, "Unexpected global option.\n");
return -1;
}
+ /*
+ * SAM format with compression enabled implies SAM.bgzf
+ */
+ if (ga->out.format == sam) {
+ hts_opt *opts = (hts_opt *)ga->out.specific;
+ while (opts) {
+ if (opts->opt == HTS_OPT_COMPRESSION_LEVEL)
+ ga->out.compression = bgzf;
+ opts = opts->next;
+ }
+ }
+
return r;
}
else if (strcmp(lopts[i].name, "threads") == 0)
fprintf(fp,"threads INT\n"
" Number of additional threads to use [0]\n");
-// else if (strcmp(lopts[i].name, "verbose") == 0)
-// fprintf(fp,"verbose\n"
-// " Increment level of verbosity\n");
+ else if (strcmp(lopts[i].name, "write-index") == 0)
+ fprintf(fp,"write-index\n"
+ " Automatically index the output files [off]\n");
+ else if (strcmp(lopts[i].name, "verbosity") == 0)
+ fprintf(fp,"verbosity INT\n"
+ " Set level of verbosity\n");
}
}
/* sam_opts.h -- utilities to aid parsing common command line options.
- Copyright (C) 2015 Genome Research Ltd.
+ Copyright (C) 2015, 2019 Genome Research Ltd.
Author: James Bonfield <jkb@sanger.ac.uk>
htsFormat out;
char *reference;
int nthreads;
- //int verbosity;
+ int write_index;
} sam_global_args;
#define SAM_GLOBAL_ARGS_INIT {{0},{0}}
SAM_OPT_OUTPUT_FMT_OPTION,
SAM_OPT_REFERENCE,
SAM_OPT_NTHREADS,
- //SAM_OPT_VERBOSE
+ SAM_OPT_WRITE_INDEX,
+ SAM_OPT_VERBOSITY,
};
#define SAM_OPT_VAL(val, defval) ((val) == '-')? '?' : (val)? (val) : (defval)
{"output-fmt", required_argument, NULL, SAM_OPT_VAL(o3, SAM_OPT_OUTPUT_FMT)}, \
{"output-fmt-option", required_argument, NULL, SAM_OPT_VAL(o4, SAM_OPT_OUTPUT_FMT_OPTION)}, \
{"reference", required_argument, NULL, SAM_OPT_VAL(o5, SAM_OPT_REFERENCE)}, \
- {"threads", required_argument, NULL, SAM_OPT_VAL(o6, SAM_OPT_NTHREADS)}
- //{"verbose", no_argument, NULL, SAM_OPT_VERBOSE}
+ {"threads", required_argument, NULL, SAM_OPT_VAL(o6, SAM_OPT_NTHREADS)}, \
+ {"write-index", no_argument, NULL, SAM_OPT_WRITE_INDEX}, \
+ {"verbosity", required_argument, NULL, SAM_OPT_VERBOSITY}
/*
* Processes a standard "global" samtools long option.
/* sam_utils.c -- various utilities internal to samtools.
- Copyright (C) 2014-2016 Genome Research Ltd.
+ Copyright (C) 2014-2016, 2018, 2019 Genome Research Ltd.
Author: John Marshall <jm18@sanger.ac.uk>
DEALINGS IN THE SOFTWARE. */
#include <config.h>
+#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
vprint_error_core(subcommand, format, args, err? strerror(err) : NULL);
va_end(args);
}
+
+void check_sam_close(const char *subcmd, samFile *fp, const char *fname, const char *null_fname, int *retp)
+{
+ int r = sam_close(fp);
+ if (r >= 0) return;
+
+ // TODO Need error infrastructure so we can print a message instead of r
+ if (fname) print_error(subcmd, "error closing \"%s\": %d", fname, r);
+ else print_error(subcmd, "error closing %s: %d", null_fname, r);
+
+ *retp = EXIT_FAILURE;
+}
+
+/* Pick an index suffix based on the output file descriptor type. */
+static char *idx_suffix(htsFile *fp) {
+ switch (fp->format.format) {
+ case sam:
+ case bam:
+ // Tough cheese if you wanted bai!
+ // New feature => mandatory new index too, for simplicity of CLI.
+ return "csi";
+
+ case cram:
+ return "crai";
+
+ default:
+ return NULL;
+ }
+}
+
+/*
+ * Utility function to add an index to a file we've opened for write.
+ * NB: Call this after writing the header and before writing sequences.
+ *
+ * The returned index filename should be freed by the caller, but only
+ * after sam_idx_save has been called.
+ *
+ * Returns index filename on success,
+ * NULL on failure.
+ */
+char *auto_index(htsFile *fp, const char *fn, bam_hdr_t *header) {
+ char *fn_idx;
+ int min_shift = 14; /* CSI */
+ if (!fn || !*fn || strcmp(fn, "-") == 0)
+ return NULL;
+
+ char *delim = strstr(fn, HTS_IDX_DELIM);
+ if (delim != NULL) {
+ delim += strlen(HTS_IDX_DELIM);
+
+ fn_idx = strdup(delim);
+ if (!fn_idx)
+ return NULL;
+
+ size_t l = strlen(fn_idx);
+ if (l >= 4 && strcmp(fn_idx + l - 4, ".bai") == 0)
+ min_shift = 0;
+ } else {
+ char *suffix = idx_suffix(fp);
+ if (!suffix)
+ return NULL;
+
+ fn_idx = malloc(strlen(fn)+6);
+ if (!fn_idx)
+ return NULL;
+
+ sprintf(fn_idx, "%s.%s", fn, suffix);
+ }
+
+ if (sam_idx_init(fp, header, min_shift, fn_idx) < 0) {
+ print_error_errno("auto_index", "failed to open index \"%s\" for writing", fn_idx);
+ free(fn_idx);
+ return NULL;
+ }
+
+ return fn_idx;
+}
/* sam_utils.c -- various utilities internal to samtools.
- Copyright (C) 2014-2016 Genome Research Ltd.
+ Copyright (C) 2014-2016, 2018, 2019 Genome Research Ltd.
Author: John Marshall <jm18@sanger.ac.uk>
DEALINGS IN THE SOFTWARE. */
#include <config.h>
+#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
vprint_error_core(subcommand, format, args, err? strerror(err) : NULL);
va_end(args);
}
+
+void check_sam_close(const char *subcmd, samFile *fp, const char *fname, const char *null_fname, int *retp)
+{
+ int r = sam_close(fp);
+ if (r >= 0) return;
+
+ // TODO Need error infrastructure so we can print a message instead of r
+ if (fname) print_error(subcmd, "error closing \"%s\": %d", fname, r);
+ else print_error(subcmd, "error closing %s: %d", null_fname, r);
+
+ *retp = EXIT_FAILURE;
+}
+
+/* Pick an index suffix based on the output file descriptor type. */
+static char *idx_suffix(htsFile *fp) {
+ switch (fp->format.format) {
+ case sam:
+ case bam:
+ // Tough cheese if you wanted bai!
+ // New feature => mandatory new index too, for simplicity of CLI.
+ return "csi";
+
+ case cram:
+ return "crai";
+
+ default:
+ return NULL;
+ }
+}
+
+/*
+ * Utility function to add an index to a file we've opened for write.
+ * NB: Call this after writing the header and before writing sequences.
+ *
+ * The returned index filename should be freed by the caller, but only
+ * after sam_idx_save has been called.
+ *
+ * Returns index filename on success,
+ * NULL on failure.
+ */
+char *auto_index(htsFile *fp, const char *fn, bam_hdr_t *header) {
+ char *fn_idx;
+ int min_shift = 14; /* CSI */
+ if (!fn || !*fn || strcmp(fn, "-") == 0)
+ return NULL;
+
+ char *delim = strstr(fn, HTS_IDX_DELIM);
+ if (delim != NULL) {
+ delim += strlen(HTS_IDX_DELIM);
+
+ fn_idx = strdup(delim);
+ if (!fn_idx)
+ return NULL;
+
+ size_t l = strlen(fn_idx);
+ if (l >= 4 && strcmp(fn_idx + l - 4, ".bai") == 0)
+ min_shift = 0;
+ } else {
+ char *suffix = idx_suffix(fp);
+ if (!suffix)
+ return NULL;
+
+ fn_idx = malloc(strlen(fn)+6);
+ if (!fn_idx)
+ return NULL;
+
+ sprintf(fn_idx, "%s.%s", fn, suffix);
+ }
+
+ if (sam_idx_init(fp, header, min_shift, fn_idx) < 0) {
+ print_error_errno("auto_index", "failed to open index \"%s\" for writing", fn_idx);
+ free(fn_idx);
+ return NULL;
+ }
+
+ return fn_idx;
+}
/* sam_view.c -- SAM<->BAM<->CRAM conversion.
- Copyright (C) 2009-2017 Genome Research Ltd.
+ Copyright (C) 2009-2019 Genome Research Ltd.
Portions copyright (C) 2009, 2011, 2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include <unistd.h>
#include <math.h>
#include <inttypes.h>
-#include <stdbool.h>
-#include <assert.h>
#include <getopt.h>
-#include <ctype.h>
#include "htslib/sam.h"
#include "htslib/faidx.h"
-#include "htslib/kstring.h"
#include "htslib/khash.h"
-#include "htslib/klist.h"
#include "htslib/thread_pool.h"
-#include "htslib/bgzf.h"
#include "samtools.h"
#include "sam_opts.h"
#include "bedidx.h"
-#define DEFAULT_BARCODE_TAG "BC"
-#define DEFAULT_QUALITY_TAG "QT"
-
KHASH_SET_INIT_STR(rg)
-#define taglist_free(p)
-KLIST_INIT(ktaglist, char*, taglist_free)
+KHASH_SET_INIT_STR(tv)
typedef khash_t(rg) *rghash_t;
+typedef khash_t(tv) *tvhash_t;
// This structure contains the settings for a samview run
typedef struct samview_settings {
rghash_t rghash;
+ tvhash_t tvhash;
int min_mapQ;
int flag_on;
int flag_off;
size_t remove_aux_len;
char** remove_aux;
int multi_region;
+ char* tag;
} samview_settings_t;
// TODO Add declarations of these to a viable htslib or samtools header
-extern const char *bam_get_library(bam_hdr_t *header, const bam1_t *b);
+extern const char *bam_get_library(sam_hdr_t *header, const bam1_t *b);
extern int bam_remove_B(bam1_t *b);
extern char *samfaipath(const char *fn_ref);
// Returns 0 to indicate read should be output 1 otherwise
-static int process_aln(const bam_hdr_t *h, bam1_t *b, samview_settings_t* settings)
+static int process_aln(const sam_hdr_t *h, bam1_t *b, samview_settings_t* settings)
{
if (settings->remove_B) bam_remove_B(b);
if (settings->min_qlen > 0) {
return 1;
if (settings->flag_alloff && ((b->core.flag & settings->flag_alloff) == settings->flag_alloff))
return 1;
- if (!settings->multi_region && settings->bed && (b->core.tid < 0 || !bed_overlap(settings->bed, h->target_name[b->core.tid], b->core.pos, bam_endpos(b))))
+ if (!settings->multi_region && settings->bed && (b->core.tid < 0 || !bed_overlap(settings->bed, sam_hdr_tid2name(h, b->core.tid), b->core.pos, bam_endpos(b))))
return 1;
if (settings->subsam_frac > 0.) {
uint32_t k = __ac_Wang_hash(__ac_X31_hash_string(bam_get_qname(b)) ^ settings->subsam_seed);
if (k == kh_end(settings->rghash)) return 1;
}
}
+ if (settings->tvhash && settings->tag) {
+ uint8_t *s = bam_aux_get(b, settings->tag);
+ if (s) {
+ khint_t k = kh_get(tv, settings->tvhash, (char*)(s + 1));
+ if (k == kh_end(settings->tvhash)) return 1;
+ } else {
+ return 1;
+ }
+ }
if (settings->library) {
- const char *p = bam_get_library((bam_hdr_t*)h, b);
+ const char *p = bam_get_library((sam_hdr_t*)h, b);
if (!p || strcmp(p, settings->library) != 0) return 1;
}
if (settings->remove_aux_len) {
return 0;
}
-static char *drop_rg(char *hdtxt, rghash_t h, int *len)
-{
- char *p = hdtxt, *q, *r, *s;
- kstring_t str;
- memset(&str, 0, sizeof(kstring_t));
- while (1) {
- int toprint = 0;
- q = strchr(p, '\n');
- if (q == 0) q = p + strlen(p);
- if (q - p < 3) break; // the line is too short; then stop
- if (strncmp(p, "@RG\t", 4) == 0) {
- int c;
- khint_t k;
- if ((r = strstr(p, "\tID:")) != 0) {
- r += 4;
- for (s = r; *s != '\0' && *s != '\n' && *s != '\t'; ++s);
- c = *s; *s = '\0';
- k = kh_get(rg, h, r);
- *s = c;
- if (k != kh_end(h)) toprint = 1;
- }
- } else toprint = 1;
- if (toprint) {
- kputsn(p, q - p, &str); kputc('\n', &str);
- }
- p = q + 1;
- }
- *len = str.l;
- return str.s;
-}
-
static int usage(FILE *fp, int exit_status, int is_long_help);
static int add_read_group_single(const char *subcmd, samview_settings_t *settings, char *name)
return (ret != -1) ? 0 : -1;
}
-static inline int check_sam_write1(samFile *fp, const bam_hdr_t *h, const bam1_t *b, const char *fname, int *retp)
+static int add_tag_value_single(const char *subcmd, samview_settings_t *settings, char *name)
{
- int r = sam_write1(fp, h, b);
- if (r >= 0) return r;
+ char *d = strdup(name);
+ int ret = 0;
- if (fname) print_error_errno("view", "writing to \"%s\" failed", fname);
- else print_error_errno("view", "writing to standard output failed");
+ if (d == NULL) goto err;
- *retp = EXIT_FAILURE;
- return r;
+ if (settings->tvhash == NULL) {
+ settings->tvhash = kh_init(tv);
+ if (settings->tvhash == NULL) goto err;
+ }
+
+ kh_put(tv, settings->tvhash, d, &ret);
+ if (ret == -1) goto err;
+ if (ret == 0) free(d); /* Duplicate */
+ return 0;
+
+ err:
+ print_error(subcmd, "Couldn't add \"%s\" to tag values list: memory exhausted?", name);
+ free(d);
+ return -1;
}
-static void check_sam_close(const char *subcmd, samFile *fp, const char *fname, const char *null_fname, int *retp)
+static int add_tag_values_file(const char *subcmd, samview_settings_t *settings, char *fn)
{
- int r = sam_close(fp);
- if (r >= 0) return;
+ FILE *fp;
+ char buf[1024];
+ int ret = 0;
+ if (settings->tvhash == NULL) {
+ settings->tvhash = kh_init(tv);
+ if (settings->tvhash == NULL) {
+ perror(NULL);
+ return -1;
+ }
+ }
- // TODO Need error infrastructure so we can print a message instead of r
- if (fname) print_error(subcmd, "error closing \"%s\": %d", fname, r);
- else print_error(subcmd, "error closing %s: %d", null_fname, r);
+ fp = fopen(fn, "r");
+ if (fp == NULL) {
+ print_error_errno(subcmd, "failed to open \"%s\" for reading", fn);
+ return -1;
+ }
+
+ while (ret != -1 && !feof(fp) && fscanf(fp, "%1023s", buf) > 0) {
+ char *d = strdup(buf);
+ if (d != NULL) {
+ kh_put(tv, settings->tvhash, d, &ret);
+ if (ret == 0) free(d); /* Duplicate */
+ } else {
+ ret = -1;
+ }
+ }
+ if (ferror(fp)) ret = -1;
+ if (ret == -1) {
+ print_error_errno(subcmd, "failed to read \"%s\"", fn);
+ }
+ fclose(fp);
+ return (ret != -1) ? 0 : -1;
+}
+
+static inline int check_sam_write1(samFile *fp, const sam_hdr_t *h, const bam1_t *b, const char *fname, int *retp)
+{
+ int r = sam_write1(fp, h, b);
+ if (r >= 0) return r;
+
+ if (fname) print_error_errno("view", "writing to \"%s\" failed", fname);
+ else print_error_errno("view", "writing to standard output failed");
*retp = EXIT_FAILURE;
+ return r;
}
int main_samview(int argc, char *argv[])
{
- int c, is_header = 0, is_header_only = 0, ret = 0, compress_level = -1, is_count = 0;
+ int c, is_header = 0, is_header_only = 0, ret = 0, compress_level = -1, is_count = 0, has_index_file = 0, no_pg = 0;
int64_t count = 0;
samFile *in = 0, *out = 0, *un_out=0;
FILE *fp_out = NULL;
- bam_hdr_t *header = NULL;
+ sam_hdr_t *header = NULL;
char out_mode[5], out_un_mode[5], *out_format = "";
- char *fn_in = 0, *fn_out = 0, *fn_list = 0, *q, *fn_un_out = 0;
+ char *fn_in = 0, *fn_idx_in = 0, *fn_out = 0, *fn_list = 0, *q, *fn_un_out = 0;
+ char *fn_out_idx = NULL, *fn_un_out_idx = NULL, *arg_list = NULL;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
htsThreadPool p = {NULL, 0};
int filter_state = ALL, filter_op = 0;
samview_settings_t settings = {
.rghash = NULL,
+ .tvhash = NULL,
.min_mapQ = 0,
.flag_on = 0,
.flag_off = 0,
.subsam_frac = -1.,
.library = NULL,
.bed = NULL,
- .multi_region = 0
+ .multi_region = 0,
+ .tag = NULL
};
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 'T', '@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
opterr = 0;
while ((c = getopt_long(argc, argv,
- "SbBcCt:h1Ho:O:q:f:F:G:ul:r:T:R:L:s:@:m:x:U:M",
+ "SbBcCt:h1Ho:O:q:f:F:G:ul:r:T:R:d:D:L:s:@:m:x:U:MX",
lopts, NULL)) >= 0) {
switch (c) {
case 's':
srand(settings.subsam_seed);
settings.subsam_seed = rand();
}
-
if (q && *q == '.') {
settings.subsam_frac = strtod(q, &q);
if (*q) ret = 1;
case 'H': is_header_only = 1; break;
case 'o': fn_out = strdup(optarg); break;
case 'U': fn_un_out = strdup(optarg); break;
+ case 'X': has_index_file = 1; break;
case 'f': settings.flag_on |= strtol(optarg, 0, 0); break;
case 'F': settings.flag_off |= strtol(optarg, 0, 0); break;
case 'G': settings.flag_alloff |= strtol(optarg, 0, 0); break;
ret = 1;
goto view_end;
}
+ break;
+ case 'd':
+ if (strlen(optarg) < 4 || optarg[2] != ':') {
+ print_error_errno("view", "Invalid \"tag:value\" option: \"%s\"", optarg);
+ ret = 1;
+ goto view_end;
+ }
+
+ if (settings.tag) {
+ if (settings.tag[0] != optarg[0] || settings.tag[1] != optarg[1]) {
+ print_error("view", "Different tag \"%s\" was specified before: \"%s\"", settings.tag, optarg);
+ ret = 1;
+ goto view_end;
+ }
+ } else {
+ if (!(settings.tag = calloc(3, 1))) {
+ print_error("view", "Could not allocate memory for tag: \"%s\"", optarg);
+ ret = 1;
+ goto view_end;
+ }
+ memcpy(settings.tag, optarg, 2);
+ }
+
+ if (add_tag_value_single("view", &settings, optarg+3) != 0) {
+ ret = 1;
+ goto view_end;
+ }
+ break;
+ case 'D':
+ // Allow ";" as delimiter besides ":" to support MinGW CLI POSIX
+ // path translation as described at:
+ // http://www.mingw.org/wiki/Posix_path_conversion
+ if (strlen(optarg) < 4 || (optarg[2] != ':' && optarg[2] != ';')) {
+ print_error_errno("view", "Invalid \"tag:file\" option: \"%s\"", optarg);
+ ret = 1;
+ goto view_end;
+ }
+
+ if (settings.tag) {
+ if (settings.tag[0] != optarg[0] || settings.tag[1] != optarg[1]) {
+ print_error("view", "Different tag \"%s\" was specified before: \"%s\"", settings.tag, optarg);
+ ret = 1;
+ goto view_end;
+ }
+ } else {
+ if (!(settings.tag = calloc(3, 1))) {
+ print_error("view", "Could not allocate memory for tag: \"%s\"", optarg);
+ ret = 1;
+ goto view_end;
+ }
+ memcpy(settings.tag, optarg, 2);
+ }
+
+ if (add_tag_values_file("view", &settings, optarg+3) != 0) {
+ ret = 1;
+ goto view_end;
+ }
break;
/* REMOVED as htslib doesn't support this
//case 'x': out_format = "x"; break;
}
break;
case 'M': settings.multi_region = 1; break;
+ case 1: no_pg = 1; break;
default:
if (parse_sam_global_opt(c, optarg, lopts, &ga) != 0)
return usage(stderr, EXIT_FAILURE, 0);
ret = 1;
goto view_end;
}
- if (settings.rghash) { // FIXME: I do not know what "bam_header_t::n_text" is for...
- char *tmp;
- int l;
- tmp = drop_rg(header->text, settings.rghash, &l);
- free(header->text);
- header->text = tmp;
- header->l_text = l;
+ if (settings.rghash) {
+ sam_hdr_remove_lines(header, "RG", "ID", settings.rghash);
}
if (!is_count) {
if ((out = sam_open_format(fn_out? fn_out : "-", out_mode, &ga.out)) == 0) {
goto view_end;
}
}
- if (*out_format || is_header ||
+
+ if (!no_pg) {
+ if (!(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("view", "failed to create arg_list");
+ ret = 1;
+ goto view_end;
+ }
+ if (sam_hdr_add_pg(header, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error("view", "failed to add PG line to the header");
+ ret = 1;
+ goto view_end;
+ }
+ }
+
+ if (*out_format || ga.write_index || is_header ||
out_mode[1] == 'b' || out_mode[1] == 'c' ||
(ga.out.format != sam && ga.out.format != unknown_format)) {
if (sam_hdr_write(out, header) != 0) {
goto view_end;
}
}
+ if (ga.write_index) {
+ if (!(fn_out_idx = auto_index(out, fn_out, header))) {
+ ret = 1;
+ goto view_end;
+ }
+ }
+
if (fn_un_out) {
if ((un_out = sam_open_format(fn_un_out, out_un_mode, &ga.out)) == 0) {
print_error_errno("view", "failed to open \"%s\" for writing", fn_un_out);
goto view_end;
}
}
+ if (ga.write_index) {
+ if (!(fn_un_out_idx = auto_index(un_out, fn_un_out, header))) {
+ ret = 1;
+ goto view_end;
+ }
+ }
}
}
else {
}
if (is_header_only) goto view_end; // no need to print alignments
+ if (has_index_file) {
+ fn_idx_in = (optind+1 < argc)? argv[optind+1] : 0;
+ if (fn_idx_in == 0) {
+ fprintf(stderr, "[main_samview] incorrect number of arguments for -X option. Aborting.\n");
+ return 1;
+ }
+ }
+
if (settings.multi_region) {
- if (optind < argc - 1) { //regions have been specified in the command line
+ if (!has_index_file && optind < argc - 1) { //regions have been specified in the command line
settings.bed = bed_hash_regions(settings.bed, argv, optind+1, argc, &filter_op); //insert(1) or filter out(0) the regions from the command line in the same hash table as the bed file
if (!filter_op)
filter_state = FILTERED;
+ } else if (has_index_file && optind < argc - 2) {
+ settings.bed = bed_hash_regions(settings.bed, argv, optind+2, argc, &filter_op); //insert(1) or filter out(0) the regions from the command line in the same hash table as the bed file
+ if (!filter_op)
+ filter_state = FILTERED;
} else {
bed_unify(settings.bed);
}
if (settings.bed == NULL) { // index is unavailable or no regions have been specified
fprintf(stderr, "[main_samview] no regions or BED file have been provided. Aborting.\n");
} else {
- hts_idx_t *idx = sam_index_load(in, fn_in); // load index
+ hts_idx_t *idx = NULL;
+ // If index filename has not been specfied, look in BAM folder
+ if (fn_idx_in != 0) {
+ idx = sam_index_load2(in, fn_in, fn_idx_in); // load index
+ } else {
+ idx = sam_index_load(in, fn_in);
+ }
if (idx != NULL) {
int regcount = 0;
}
bam_destroy1(b);
} else {
- if (optind + 1 >= argc) { // convert/print the entire file
+ if ((has_index_file && optind >= argc - 2) || (!has_index_file && optind >= argc - 1)) { // convert/print the entire file
bam1_t *b = bam_init1();
int r;
while ((r = sam_read1(in, header, b)) >= 0) { // read one alignment from `in'
} else { // retrieve alignments in specified regions
int i;
bam1_t *b;
- hts_idx_t *idx = sam_index_load(in, fn_in); // load index
+ hts_idx_t *idx = NULL;
+ // If index filename has not been specfied, look in BAM folder
+ if (fn_idx_in != NULL) {
+ idx = sam_index_load2(in, fn_in, fn_idx_in); // load index
+ } else {
+ idx = sam_index_load(in, fn_in);
+ }
if (idx == 0) { // index is unavailable
fprintf(stderr, "[main_samview] random alignment retrieval only works for indexed BAM or CRAM files.\n");
ret = 1;
goto view_end;
}
b = bam_init1();
- for (i = optind + 1; i < argc; ++i) {
+
+ for (i = (has_index_file)? optind+2 : optind+1; i < argc; ++i) {
int result;
hts_itr_t *iter = sam_itr_querys(idx, header, argv[i]); // parse a region in the format like `chr2:100-200'
if (iter == NULL) { // region invalid or reference name not found
- int beg, end;
- if (hts_parse_reg(argv[i], &beg, &end))
- fprintf(stderr, "[main_samview] region \"%s\" specifies an unknown reference name. Continue anyway.\n", argv[i]);
- else
- fprintf(stderr, "[main_samview] region \"%s\" could not be parsed. Continue anyway.\n", argv[i]);
+ fprintf(stderr, "[main_samview] region \"%s\" specifies an invalid region or unknown reference. Continue anyway.\n", argv[i]);
continue;
}
// fetch alignments
}
}
+ if (ga.write_index) {
+ if (sam_idx_save(out) < 0) {
+ print_error_errno("view", "writing index failed");
+ ret = 1;
+ }
+ if (un_out && sam_idx_save(un_out) < 0) {
+ print_error_errno("view", "writing index failed");
+ ret = 1;
+ }
+ }
+
view_end:
if (is_count && ret == 0) {
if (fprintf(fn_out? fp_out : stdout, "%" PRId64 "\n", count) < 0) {
free(fn_list); free(fn_out); free(settings.library); free(fn_un_out);
sam_global_args_free(&ga);
- if ( header ) bam_hdr_destroy(header);
+ if ( header ) sam_hdr_destroy(header);
if (settings.bed) bed_destroy(settings.bed);
if (settings.rghash) {
khint_t k;
if (kh_exist(settings.rghash, k)) free((char*)kh_key(settings.rghash, k));
kh_destroy(rg, settings.rghash);
}
+ if (settings.tvhash) {
+ khint_t k;
+ for (k = 0; k < kh_end(settings.tvhash); ++k)
+ if (kh_exist(settings.tvhash, k)) free((char*)kh_key(settings.tvhash, k));
+ kh_destroy(tv, settings.tvhash);
+ }
if (settings.remove_aux_len) {
free(settings.remove_aux);
}
+ if (settings.tag) {
+ free(settings.tag);
+ }
if (p.pool)
hts_tpool_destroy(p.pool);
+ if (fn_out_idx)
+ free(fn_out_idx);
+ if (fn_un_out_idx)
+ free(fn_un_out_idx);
+ free(arg_list);
+
return ret;
}
" -U FILE output reads not selected by filters to FILE [null]\n"
// extra input
" -t FILE FILE listing reference names and lengths (see long help) [null]\n"
+" -X include customized index file\n"
// read filters
" -L FILE only include reads overlapping this BED FILE [null]\n"
" -r STR only include reads in read group STR [null]\n"
" -R FILE only include reads with read group listed in FILE [null]\n"
+" -d STR:STR\n"
+" only include reads with tag STR and associated value STR [null]\n"
+" -D STR:FILE\n"
+" only include reads with tag STR and associated values listed in\n"
+" FILE [null]\n"
" -q INT only include reads with mapping quality >= INT [0]\n"
" -l STR only include reads in library STR [null]\n"
" -m INT only include reads with number of CIGAR operations consuming\n"
" -B collapse the backward CIGAR operation\n"
// general options
" -? print long help, including note about region specification\n"
-" -S ignored (input format is auto-detected)\n");
+" -S ignored (input format is auto-detected)\n"
+" --no-PG do not add a PG line\n");
- sam_global_opt_help(fp, "-.O.T@");
+ sam_global_opt_help(fp, "-.O.T@..");
fprintf(fp, "\n");
if (is_long_help)
free(argv2);
return ret;
}
-
-int8_t seq_comp_table[16] = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 };
-static const char *copied_tags[] = { "RG", "BC", "QT", NULL };
-
-static void bam2fq_usage(FILE *to, const char *command)
-{
- int fq = strcasecmp("fastq", command) == 0 || strcasecmp("bam2fq", command) == 0;
- fprintf(to,
-"Usage: samtools %s [options...] <in.bam>\n", command);
- fprintf(to,
-"Options:\n"
-" -0 FILE write reads designated READ_OTHER to FILE\n"
-" -1 FILE write reads designated READ1 to FILE\n"
-" -2 FILE write reads designated READ2 to FILE\n"
-" note: if a singleton file is specified with -s, only\n"
-" paired reads will be written to the -1 and -2 files.\n"
-" -f INT only include reads with all of the FLAGs in INT present [0]\n" // F&x == x
-" -F INT only include reads with none of the FLAGS in INT present [0]\n" // F&x == 0
-" -G INT only EXCLUDE reads with all of the FLAGs in INT present [0]\n" // !(F&x == x)
-" -n don't append /1 and /2 to the read name\n"
-" -N always append /1 and /2 to the read name\n");
- if (fq) fprintf(to,
-" -O output quality in the OQ tag if present\n");
- fprintf(to,
-" -s FILE write singleton reads designated READ1 or READ2 to FILE\n"
-" -t copy RG, BC and QT tags to the %s header line\n",
- fq ? "FASTQ" : "FASTA");
- fprintf(to,
-" -T TAGLIST copy arbitrary tags to the %s header line\n",
- fq ? "FASTQ" : "FASTA");
- if (fq) fprintf(to,
-" -v INT default quality score if not given in file [1]\n"
-" -i add Illumina Casava 1.8 format entry to header (eg 1:N:0:ATCACG)\n"
-" -c compression level [0..9] to use when creating gz or bgzf fastq files\n"
-" --i1 FILE write first index reads to FILE\n"
-" --i2 FILE write second index reads to FILE\n"
-" --barcode-tag TAG Barcode tag [default: " DEFAULT_BARCODE_TAG "]\n"
-" --quality-tag TAG Quality tag [default: " DEFAULT_QUALITY_TAG "]\n"
-" --index-format STR How to parse barcode and quality tags\n\n");
- sam_global_opt_help(to, "-.--.@");
- fprintf(to,
-"\n"
-"Reads are designated READ1 if FLAG READ1 is set and READ2 is not set.\n"
-"Reads are designated READ2 if FLAG READ1 is not set and READ2 is set.\n"
-"Reads are designated READ_OTHER if FLAGs READ1 and READ2 are either both set\n"
-"or both unset.\n"
-"Run 'samtools flags' for more information on flag codes and meanings.\n");
- fprintf(to,
-"\n"
-"The index-format string describes how to parse the barcode and quality tags, for example:\n"
-" i14i8 the first 14 characters are index 1, the next 8 characters are index 2\n"
-" n8i14 ignore the first 8 characters, and use the next 14 characters for index 1\n"
-"If the tag contains a separator, then the numeric part can be replaced with '*' to mean\n"
-"'read until the separator or end of tag', for example:\n"
-" n*i* ignore the left part of the tag until the separator, then use the second part\n"
-" of the tag as index 1\n");
- fprintf(to,
-"\n"
-"Examples:\n"
-" To get just the paired reads in separate files, use:\n"
-" samtools %s -1 paired1.%s -2 paired2.%s -0 /dev/null -s /dev/null -n -F 0x900 in.bam\n"
-"\n To get all non-supplementary/secondary reads in a single file, redirect the output:\n"
-" samtools %s -F 0x900 in.bam > all_reads.%s\n",
- command, fq ? "fq" : "fa", fq ? "fq" : "fa",
- command, fq ? "fq" : "fa");
-}
-
-typedef enum { READ_UNKNOWN = 0, READ_1 = 1, READ_2 = 2 } readpart;
-typedef enum { FASTA, FASTQ } fastfile;
-typedef struct bam2fq_opts {
- char *fnse;
- char *fnr[3];
- char *fn_input; // pointer to input filename in argv do not free
- bool has12, has12always, use_oq, copy_tags, illumina_tag;
- int flag_on, flag_off, flag_alloff;
- sam_global_args ga;
- fastfile filetype;
- int def_qual;
- char *barcode_tag;
- char *quality_tag;
- char *index_file[2];
- char *index_format;
- char *extra_tags;
- char compression_level;
-} bam2fq_opts_t;
-
-typedef struct bam2fq_state {
- samFile *fp;
- BGZF *fpse;
- BGZF *fpr[3];
- BGZF *fpi[2];
- BGZF *hstdout;
- bam_hdr_t *h;
- bool has12, use_oq, copy_tags, illumina_tag;
- int flag_on, flag_off, flag_alloff;
- fastfile filetype;
- int def_qual;
- klist_t(ktaglist) *taglist;
- char *index_sequence;
- char compression_level;
-} bam2fq_state_t;
-
-/*
- * Get and decode the read from a BAM record.
- *
- * TODO: htslib really needs an interface for this. Consider this or perhaps
- * bam_get_seq_str (current vs original orientation) and bam_get_qual_str
- * functions as string formatted equivalents to bam_get_{seq,qual}?
- */
-
-/*
- * Reverse a string in place.
- * From http://stackoverflow.com/questions/8534274/is-the-strrev-function-not-available-in-linux.
- * Author Sumit-naik: http://stackoverflow.com/users/4590926/sumit-naik
- */
-static char *reverse(char *str)
-{
- int i = strlen(str)-1,j=0;
- char ch;
- while (i>j) {
- ch = str[i];
- str[i]= str[j];
- str[j] = ch;
- i--;
- j++;
- }
- return str;
-}
-
-/* return the read, reverse complemented if necessary */
-static char *get_read(const bam1_t *rec)
-{
- int len = rec->core.l_qseq + 1;
- char *read = calloc(1, len);
- char *seq = (char *)bam_get_seq(rec);
- int n;
-
- if (!read) return NULL;
-
- for (n=0; n < rec->core.l_qseq; n++) {
- if (rec->core.flag & BAM_FREVERSE) read[n] = seq_nt16_str[seq_comp_table[bam_seqi(seq,n)]];
- else read[n] = seq_nt16_str[bam_seqi(seq,n)];
- }
- if (rec->core.flag & BAM_FREVERSE) reverse(read);
- return read;
-}
-
-/*
- * get and decode the quality from a BAM record
- */
-static int get_quality(const bam1_t *rec, char **qual_out)
-{
- char *quality = calloc(1, rec->core.l_qseq + 1);
- char *q = (char *)bam_get_qual(rec);
- int n;
-
- if (!quality) return -1;
-
- if (*q == '\xff') {
- free(quality);
- *qual_out = NULL;
- return 0;
- }
-
- for (n=0; n < rec->core.l_qseq; n++) {
- quality[n] = q[n]+33;
- }
- if (rec->core.flag & BAM_FREVERSE) reverse(quality);
- *qual_out = quality;
- return 0;
-}
-
-//
-// End of htslib complaints
-//
-
-
-static readpart which_readpart(const bam1_t *b)
-{
- if ((b->core.flag & BAM_FREAD1) && !(b->core.flag & BAM_FREAD2)) {
- return READ_1;
- } else if ((b->core.flag & BAM_FREAD2) && !(b->core.flag & BAM_FREAD1)) {
- return READ_2;
- } else {
- return READ_UNKNOWN;
- }
-}
-
-/*
- * parse the length part from the index-format string
- */
-static int getLength(char **s)
-{
- int n = 0;
- while (**s) {
- if (**s == '*') { n=-1; (*s)++; break; }
- if ( !isdigit(**s)) break;
- n = n*10 + ((**s)-'0');
- (*s)++;
- }
- return n;
-}
-
-static bool copy_tag(const char *tag, const bam1_t *rec, kstring_t *linebuf)
-{
- uint8_t *s = bam_aux_get(rec, tag);
- if (s) {
- char aux_type = *s;
- switch (aux_type) {
- case 'C':
- case 'S': aux_type = 'I'; break;
- case 'c':
- case 's': aux_type = 'i'; break;
- case 'd': aux_type = 'f'; break;
- }
-
- // Ensure space. Need 6 chars + length of tag. Max length of
- // i is 16, A is 21, B currently 26, Z is unknown, so
- // have to check that one later.
- if (ks_resize(linebuf, ks_len(linebuf) + 64) < 0) return false;
-
- kputc('\t', linebuf);
- kputsn(tag, 2, linebuf);
- kputc(':', linebuf);
- kputc(aux_type=='I'? 'i': aux_type, linebuf);
- kputc(':', linebuf);
- switch (aux_type) {
- case 'H':
- case 'Z':
- if (kputs(bam_aux2Z(s), linebuf) < 0) return false;
- break;
- case 'i': kputw(bam_aux2i(s), linebuf); break;
- case 'I': kputuw(bam_aux2i(s), linebuf); break;
- case 'A': kputc(bam_aux2A(s), linebuf); break;
- case 'f': kputd(bam_aux2f(s), linebuf); break;
- case 'B': kputs("*** Unhandled aux type ***", linebuf); return false;
- default: kputs("*** Unknown aux type ***", linebuf); return false;
- }
- }
- return true;
-}
-
-static int insert_index_sequence_into_linebuf(char *index_sequence, kstring_t *linebuf, bam1_t *rec)
-{
- if (!index_sequence) return 0;
-
- kstring_t new = {0,0,NULL};
- if (linebuf->s) {
- char *s = strchr(linebuf->s, '\n');
- if (s) {
- if (ks_resize(&new, linebuf->l + strlen(index_sequence) + 16) < 0)
- return -1;
- *s = 0;
- kputs(linebuf->s, &new);
- kputc(' ', &new);
- readpart readpart = which_readpart(rec);
- if (readpart == READ_1) kputc('1', &new);
- else if (readpart == READ_2) kputc('2', &new);
- else kputc('0', &new);
-
- kputc(':', &new);
- if (rec->core.flag & BAM_FQCFAIL) kputc('Y', &new);
- else kputc('N', &new);
-
- kputs(":0:", &new);
- kputs(index_sequence, &new);
- kputc('\n', &new);
- kputs(s+1, &new);
- free(ks_release(linebuf));
- linebuf->s = new.s; linebuf->l = new.l; linebuf->m = new.m;
- }
- }
- return 0;
-}
-
-static bool make_fq_line(const bam1_t *rec, char *seq, char *qual, kstring_t *linebuf, const bam2fq_state_t *state)
-{
- int i;
-
- linebuf->l = 0;
- // Write read name
- if (kputc(state->filetype == FASTA? '>' : '@', linebuf) < 0) return false;
- if (kputs(bam_get_qname(rec), linebuf) < 0) return false;
- // Add the /1 /2 if requested
- if (state->has12) {
- readpart readpart = which_readpart(rec);
- if (readpart == READ_1) {
- if (kputs("/1", linebuf) < 0) return false;
- } else if (readpart == READ_2) {
- if (kputs("/2", linebuf) < 0) return false;
- }
- }
- if (state->copy_tags) {
- for (i = 0; copied_tags[i]; ++i) {
- if (!copy_tag(copied_tags[i], rec, linebuf)) {
- fprintf(stderr, "Problem copying aux tags: [%s]\n", linebuf->s);
- return false;
- }
- }
- }
-
- if (state->taglist->size) {
- kliter_t(ktaglist) *p;
- for (p = kl_begin(state->taglist); p != kl_end(state->taglist); p = kl_next(p)) {
- if (!copy_tag(kl_val(p), rec, linebuf)) {
- fprintf(stderr, "Problem copying aux tags: [%s]\n", linebuf->s);
- return false;
- }
- }
- }
-
- if (kputc('\n', linebuf) < 0) return false;
- if (kputs(seq, linebuf) < 0) return false;
- if (kputc('\n', linebuf) < 0) return false;
-
- if (state->filetype == FASTQ) {
- // Write quality
- if (kputs("+\n", linebuf) < 0) return false;
- if (qual && *qual) {
- if (kputs(qual, linebuf) < 0) return false;
- } else {
- int len = strlen(seq);
- if (ks_resize(linebuf, ks_len(linebuf) + len + 1) < 0) return false;
- for (i = 0; i < len; ++i) {
- kputc(33 + state->def_qual, linebuf);
- }
- }
- if (kputc('\n', linebuf) < 0) return false;
- }
- return true;
-}
-
-/*
- * Create FASTQ lines from the barcode tag using the index-format
- */
-static bool tags2fq(bam1_t *rec, bam2fq_state_t *state, const bam2fq_opts_t* opts)
-{
- uint8_t *p;
- char *ifmt = opts->index_format;
- char *tag = NULL;
- char *qual = NULL;
- char *sub_tag = NULL;
- char *sub_qual = NULL;
- size_t tag_len;
- int file_number = 0;
- kstring_t linebuf = { 0, 0, NULL }; // Buffer
-
-
- // read barcode tag
- p = bam_aux_get(rec,opts->barcode_tag);
- if (p) tag = bam_aux2Z(p);
-
- if (!tag) return true; // there is no tag
-
- tag_len = strlen(tag);
- sub_tag = calloc(1, tag_len + 1);
- if (!sub_tag) goto fail;
- sub_qual = calloc(1, tag_len + 1);
- if (!sub_qual) goto fail;
-
- // read quality tag
- p = bam_aux_get(rec, opts->quality_tag);
- if (p) qual = bam_aux2Z(p);
-
- // Parse the index-format string
- while (*ifmt) {
- if (file_number > 1) break; // shouldn't happen if we've validated paramaters correctly
- char action = *ifmt; // should be 'i' or 'n'
- ifmt++; // skip over action
- int index_len = getLength(&ifmt);
- int n = 0;
-
- if (index_len < 0) {
- // read until separator
- while (isalpha(*tag)) {
- sub_tag[n] = *tag++;
- if (qual) sub_qual[n] = *qual++;
- n++;
- }
- if (*tag) { // skip separator
- tag++;
- if (qual) qual++;
- }
- } else {
- // read index_len characters
- while (index_len-- && *tag) {
- sub_tag[n] = *tag++;
- if (qual) sub_qual[n] = *qual++;
- n++;
- }
- }
- sub_tag[n] = '\0';
- sub_qual[n] = '\0';
-
- if (action=='i' && *sub_tag && state->fpi[file_number]) {
- //if (file_number==0) state->index_sequence = strdup(sub_tag); // we're going to need this later...
- state->index_sequence = strdup(sub_tag); // we're going to need this later...
- if (!state->index_sequence) goto fail;
- if (!make_fq_line(rec, sub_tag, sub_qual, &linebuf, state)) goto fail;
- if (state->illumina_tag) {
- if (insert_index_sequence_into_linebuf(state->index_sequence, &linebuf, rec) < 0) {
- goto fail;
- }
- }
- if (bgzf_write(state->fpi[file_number++], linebuf.s, linebuf.l) < 0)
- goto fail;
- }
-
- }
-
- free(sub_qual); free(sub_tag);
- free(linebuf.s);
- return true;
-
- fail:
- perror(__func__);
- free(sub_qual); free(sub_tag);
- free(linebuf.s);
- return true;
-}
-
-// Transform a bam1_t record into a string with the FASTQ representation of it
-// @returns false for error, true for success
-static bool bam1_to_fq(const bam1_t *b, kstring_t *linebuf, const bam2fq_state_t *state)
-{
- int32_t qlen = b->core.l_qseq;
- assert(qlen >= 0);
- const uint8_t *oq = NULL;
- char *qual = NULL;
-
- char *seq = get_read(b);
- if (!seq) return false;
-
- if (state->use_oq) oq = bam_aux_get(b, "OQ");
- if (oq && *oq=='Z') {
- qual = strdup(bam_aux2Z(oq));
- if (!qual) goto fail;
- if (b->core.flag & BAM_FREVERSE) { // read is reverse complemented
- reverse(qual);
- }
- } else {
- if (get_quality(b, &qual) < 0) goto fail;
- }
-
- if (!make_fq_line(b, seq, qual, linebuf, state)) goto fail;
-
- free(qual);
- free(seq);
- return true;
-
- fail:
- free(seq);
- free(qual);
- return false;
-}
-
-static void free_opts(bam2fq_opts_t *opts)
-{
- free(opts->barcode_tag);
- free(opts->quality_tag);
- free(opts->index_format);
- free(opts->extra_tags);
- free(opts);
-}
-
-// return true if valid
-static bool parse_opts(int argc, char *argv[], bam2fq_opts_t** opts_out)
-{
- // Parse args
- bam2fq_opts_t* opts = calloc(1, sizeof(bam2fq_opts_t));
- opts->has12 = true;
- opts->has12always = false;
- opts->filetype = FASTQ;
- opts->def_qual = 1;
- opts->barcode_tag = NULL;
- opts->quality_tag = NULL;
- opts->index_format = NULL;
- opts->index_file[0] = NULL;
- opts->index_file[1] = NULL;
- opts->extra_tags = NULL;
- opts->compression_level = 1;
-
- int c;
- sam_global_args_init(&opts->ga);
- static const struct option lopts[] = {
- SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0, '@'),
- {"i1", required_argument, NULL, 1},
- {"I1", required_argument, NULL, 1},
- {"i2", required_argument, NULL, 2},
- {"I2", required_argument, NULL, 2},
- {"if", required_argument, NULL, 3},
- {"IF", required_argument, NULL, 3},
- {"index-format", required_argument, NULL, 3},
- {"barcode-tag", required_argument, NULL, 'b'},
- {"quality-tag", required_argument, NULL, 'q'},
- { NULL, 0, NULL, 0 }
- };
- while ((c = getopt_long(argc, argv, "0:1:2:f:F:G:niNOs:c:tT:v:@:", lopts, NULL)) > 0) {
- switch (c) {
- case 'b': opts->barcode_tag = strdup(optarg); break;
- case 'q': opts->quality_tag = strdup(optarg); break;
- case 1 : opts->index_file[0] = optarg; break;
- case 2 : opts->index_file[1] = optarg; break;
- case 3 : opts->index_format = strdup(optarg); break;
- case '0': opts->fnr[0] = optarg; break;
- case '1': opts->fnr[1] = optarg; break;
- case '2': opts->fnr[2] = optarg; break;
- case 'f': opts->flag_on |= strtol(optarg, 0, 0); break;
- case 'F': opts->flag_off |= strtol(optarg, 0, 0); break;
- case 'G': opts->flag_alloff |= strtol(optarg, 0, 0); break;
- case 'n': opts->has12 = false; break;
- case 'N': opts->has12always = true; break;
- case 'O': opts->use_oq = true; break;
- case 's': opts->fnse = optarg; break;
- case 't': opts->copy_tags = true; break;
- case 'i': opts->illumina_tag = true; break;
- case 'c': opts->compression_level = atoi(optarg); break;
- case 'T': opts->extra_tags = strdup(optarg); break;
- case 'v': opts->def_qual = atoi(optarg); break;
- case '?': bam2fq_usage(stderr, argv[0]); free_opts(opts); return false;
- default:
- if (parse_sam_global_opt(c, optarg, lopts, &opts->ga) != 0) {
- bam2fq_usage(stderr, argv[0]); free_opts(opts); return false;
- }
- break;
- }
- }
-
- if (opts->fnr[1] || opts->fnr[2]) opts->has12 = false;
- if (opts->has12always) opts->has12 = true;
-
- if (!opts->barcode_tag) opts->barcode_tag = strdup(DEFAULT_BARCODE_TAG);
- if (!opts->quality_tag) opts->quality_tag = strdup(DEFAULT_QUALITY_TAG);
-
- int nIndex = 0;
- if (opts->index_format) {
- char *s;
- for (s = opts->index_format; *s; s++) {
- if (*s == 'i') nIndex++;
- }
- }
- if (nIndex>2) {
- fprintf(stderr,"Invalid index format: more than 2 indexes\n");
- bam2fq_usage(stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if (opts->index_file[1] && !opts->index_file[0]) {
- fprintf(stderr, "Index one specified, but index two not given\n");
- bam2fq_usage(stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if (nIndex==2 && !opts->index_file[1]) {
- fprintf(stderr, "index_format specifies two indexes, but only one index file given\n");
- bam2fq_usage(stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if (nIndex==1 && !opts->index_file[0]) {
- fprintf(stderr, "index_format specifies an index, but no index file given\n");
- bam2fq_usage(stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if (nIndex==0 && opts->index_file[0]) {
- fprintf(stderr, "index_format not specified, but index file given\n");
- bam2fq_usage(stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if (opts->def_qual < 0 || 93 < opts->def_qual) {
- fprintf(stderr, "Invalid -v default quality %i, allowed range 0 to 93\n", opts->def_qual);
- bam2fq_usage(stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- const char* type_str = argv[0];
- if (strcasecmp("fastq", type_str) == 0 || strcasecmp("bam2fq", type_str) == 0) {
- opts->filetype = FASTQ;
- } else if (strcasecmp("fasta", type_str) == 0) {
- opts->filetype = FASTA;
- } else {
- print_error("bam2fq", "Unrecognised type call \"%s\", this should be impossible... but you managed it!", type_str);
- bam2fq_usage(stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if ((argc - (optind)) == 0) {
- fprintf(stderr, "No input file specified.\n");
- bam2fq_usage(stdout, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if ((argc - (optind)) != 1) {
- fprintf(stderr, "Too many arguments.\n");
- bam2fq_usage(stderr, argv[0]);
- free_opts(opts);
- return false;
- }
- opts->fn_input = argv[optind];
- *opts_out = opts;
- return true;
-}
-
-static BGZF *open_fqfile(char *filename, int c)
-{
- char mode[4] = "w";
- size_t len = strlen(filename);
-
- mode[2] = 0; mode[3] = 0;
- if (len > 3 && strstr(filename + (len - 3),".gz")) {
- mode[1] = 'g'; mode[2] = c+'0';
- } else if ((len > 4 && strstr(filename + (len - 4),".bgz"))
- || (len > 5 && strstr(filename + (len - 5),".bgzf"))) {
- mode[1] = c+'0';
- } else {
- mode[1] = 'u';
- }
-
- return bgzf_open(filename,mode);
-}
-
-static bool init_state(const bam2fq_opts_t* opts, bam2fq_state_t** state_out)
-{
- bam2fq_state_t* state = calloc(1, sizeof(bam2fq_state_t));
- state->flag_on = opts->flag_on;
- state->flag_off = opts->flag_off;
- state->flag_alloff = opts->flag_alloff;
- state->has12 = opts->has12;
- state->use_oq = opts->use_oq;
- state->illumina_tag = opts->illumina_tag;
- state->copy_tags = opts->copy_tags;
- state->filetype = opts->filetype;
- state->def_qual = opts->def_qual;
- state->index_sequence = NULL;
- state->hstdout = NULL;
- state->compression_level = opts->compression_level;
-
- state->taglist = kl_init(ktaglist);
- if (opts->extra_tags) {
- char *save_p;
- char *s = strtok_r(opts->extra_tags, ",", &save_p);
- while (s) {
- if (strlen(s) != 2) {
- fprintf(stderr, "Parsing extra tags - '%s' is not two characters\n", s);
- free(state);
- return false;
- }
- char **et = kl_pushp(ktaglist, state->taglist);
- *et = s;
- s = strtok_r(NULL, ",", &save_p);
- }
- }
-
- state->fp = sam_open(opts->fn_input, "r");
- if (state->fp == NULL) {
- print_error_errno("bam2fq","Cannot read file \"%s\"", opts->fn_input);
- free(state);
- return false;
- }
- if (opts->ga.nthreads > 0)
- hts_set_threads(state->fp, opts->ga.nthreads);
- uint32_t rf = SAM_QNAME | SAM_FLAG | SAM_SEQ | SAM_QUAL;
- if (opts->use_oq || opts->extra_tags || opts->index_file[0]) rf |= SAM_AUX;
- if (hts_set_opt(state->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
- fprintf(stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
- free(state);
- return false;
- }
- if (hts_set_opt(state->fp, CRAM_OPT_DECODE_MD, 0)) {
- fprintf(stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
- free(state);
- return false;
- }
- if (opts->fnse) {
- state->fpse = open_fqfile(opts->fnse, state->compression_level);
- if (state->fpse == NULL) {
- print_error_errno("bam2fq", "Cannot write to singleton file \"%s\"", opts->fnse);
- free(state);
- return false;
- }
- }
-
- if (opts->ga.reference) {
- if (hts_set_fai_filename(state->fp, opts->ga.reference) != 0) {
- print_error_errno("bam2fq", "cannot load reference \"%s\"", opts->ga.reference);
- free(state);
- return false;
- }
- }
-
- int i;
- for (i = 0; i < 3; ++i) {
- if (opts->fnr[i]) {
- state->fpr[i] = open_fqfile(opts->fnr[i], state->compression_level);
- if (state->fpr[i] == NULL) {
- print_error_errno("bam2fq", "Cannot write to r%d file \"%s\"", i, opts->fnr[i]);
- free(state);
- return false;
- }
- } else {
- if (!state->hstdout) {
- state->hstdout = bgzf_dopen(fileno(stdout), "wu");
- if (!state->hstdout) {
- print_error_errno("bam2fq", "Cannot open STDOUT");
- free(state);
- return false;
- }
- }
- state->fpr[i] = state->hstdout;
- }
- }
- for (i = 0; i < 2; i++) {
- state->fpi[i] = NULL;
- if (opts->index_file[i]) {
- state->fpi[i] = open_fqfile(opts->index_file[i], state->compression_level);
- if (state->fpi[i] == NULL) {
- print_error_errno("bam2fq", "Cannot write to i%d file \"%s\"", i+1, opts->index_file[i]);
- free(state);
- return false;
- }
- }
- }
-
- state->h = sam_hdr_read(state->fp);
- if (state->h == NULL) {
- fprintf(stderr, "Failed to read header for \"%s\"\n", opts->fn_input);
- free(state);
- return false;
- }
-
- *state_out = state;
- return true;
-}
-
-static bool destroy_state(const bam2fq_opts_t *opts, bam2fq_state_t *state, int* status)
-{
- bool valid = true;
- bam_hdr_destroy(state->h);
- check_sam_close("bam2fq", state->fp, opts->fn_input, "file", status);
- if (state->fpse && bgzf_close(state->fpse)) { print_error_errno("bam2fq", "Error closing singleton file \"%s\"", opts->fnse); valid = false; }
- int i;
- for (i = 0; i < 3; ++i) {
- if (state->fpr[i] != state->hstdout) {
- if (bgzf_close(state->fpr[i])) { print_error_errno("bam2fq", "Error closing r%d file \"%s\"", i, opts->fnr[i]); valid = false; }
- }
- }
- if (state->hstdout) {
- if (bgzf_close(state->hstdout)) {
- print_error_errno("bam2fq", "Error closing STDOUT");
- valid = false;
- }
- }
- for (i = 0; i < 2; i++) {
- if (state->fpi[i] && bgzf_close(state->fpi[i])) {
- print_error_errno("bam2fq", "Error closing i%d file \"%s\"", i+1, opts->index_file[i]);
- valid = false;
- }
- }
- kl_destroy(ktaglist,state->taglist);
- free(state->index_sequence);
- free(state);
- return valid;
-}
-
-static inline bool filter_it_out(const bam1_t *b, const bam2fq_state_t *state)
-{
- return (b->core.flag&(BAM_FSECONDARY|BAM_FSUPPLEMENTARY) // skip secondary and supplementary alignments
- || (b->core.flag&(state->flag_on)) != state->flag_on // or reads indicated by filter flags
- || (b->core.flag&(state->flag_off)) != 0
- || (b->core.flag&(state->flag_alloff) && (b->core.flag&(state->flag_alloff)) == state->flag_alloff));
-
-}
-
-static bool bam2fq_mainloop(bam2fq_state_t *state, bam2fq_opts_t* opts)
-{
- int n;
- bam1_t *records[3];
- bam1_t* b = bam_init1();
- char *current_qname = NULL;
- int64_t n_reads = 0, n_singletons = 0; // Statistics
- kstring_t linebuf[3] = {{0,0,NULL},{0,0,NULL},{0,0,NULL}};
- int score[3];
- int at_eof;
- if (b == NULL ) {
- perror("[bam2fq_mainloop] Malloc error for bam record buffer.");
- return false;
- }
-
- bool valid = true;
- while (true) {
- int res = sam_read1(state->fp, state->h, b);
- if (res < -1) {
- fprintf(stderr, "[bam2fq_mainloop] Failed to read bam record.\n");
- return false;
- }
- at_eof = res < 0;
-
- if (!at_eof && filter_it_out(b, state)) continue;
- if (!at_eof) ++n_reads;
-
- if (at_eof || !current_qname || (strcmp(current_qname, bam_get_qname(b)) != 0)) {
- if (current_qname) {
- if (state->illumina_tag) {
- for (n=0; valid && n<3; n++) {
- if (insert_index_sequence_into_linebuf(state->index_sequence, &linebuf[n], records[n]) < 0) valid = false;
- }
- if (!valid) break;
- }
- free(state->index_sequence); state->index_sequence = NULL;
- if (score[1] > 0 && score[2] > 0) {
- // print linebuf[1] to fpr[1], linebuf[2] to fpr[2]
- if (bgzf_write(state->fpr[1], linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
- if (bgzf_write(state->fpr[2], linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
- } else if (score[1] > 0 || score[2] > 0) {
- if (state->fpse) {
- // print whichever one exists to fpse
- if (score[1] > 0) {
- if (bgzf_write(state->fpse, linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
- } else {
- if (bgzf_write(state->fpse, linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
- }
- ++n_singletons;
- } else {
- if (score[1] > 0) {
- if (bgzf_write(state->fpr[1], linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
- } else {
- if (bgzf_write(state->fpr[2], linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
- }
- }
- }
- if (score[0]) { // TODO: check this
- // print linebuf[0] to fpr[0]
- if (bgzf_write(state->fpr[0], linebuf[0].s, linebuf[0].l) < 0) { valid = false; break; }
- }
- }
-
- if (at_eof) break;
-
- free(current_qname);
- current_qname = strdup(bam_get_qname(b));
- if (!current_qname) { valid = false; break; }
- score[0] = score[1] = score[2] = 0;
- }
-
- // Prefer a copy of the read that has base qualities
- int b_score = bam_get_qual(b)[0] != 0xff? 2 : 1;
- if (b_score > score[which_readpart(b)]) {
- if (state->fpi[0]) if (!tags2fq(b, state, opts)) return false;
- records[which_readpart(b)] = b;
- if(!bam1_to_fq(b, &linebuf[which_readpart(b)], state)) {
- fprintf(stderr, "[%s] Error converting read to FASTA/Q\n", __func__);
- return false;
- }
- score[which_readpart(b)] = b_score;
- }
- }
- if (!valid)
- {
- perror("[bam2fq_mainloop] Error writing to FASTx files.");
- }
- bam_destroy1(b);
- free(current_qname);
- free(linebuf[0].s);
- free(linebuf[1].s);
- free(linebuf[2].s);
- fprintf(stderr, "[M::%s] discarded %" PRId64 " singletons\n", __func__, n_singletons);
- fprintf(stderr, "[M::%s] processed %" PRId64 " reads\n", __func__, n_reads);
-
- return valid;
-}
-
-int main_bam2fq(int argc, char *argv[])
-{
- int status = EXIT_SUCCESS;
- bam2fq_opts_t* opts = NULL;
- bam2fq_state_t* state = NULL;
-
- bool valid = parse_opts(argc, argv, &opts);
- if (!valid || opts == NULL) return valid ? EXIT_SUCCESS : EXIT_FAILURE;
-
- if (!init_state(opts, &state)) return EXIT_FAILURE;
-
- if (!bam2fq_mainloop(state,opts)) status = EXIT_FAILURE;
-
- if (!destroy_state(opts, state, &status)) return EXIT_FAILURE;
- sam_global_args_free(&opts->ga);
- free_opts(opts);
-
- return status;
-}
/* sam_view.c -- SAM<->BAM<->CRAM conversion.
- Copyright (C) 2009-2017 Genome Research Ltd.
+ Copyright (C) 2009-2019 Genome Research Ltd.
Portions copyright (C) 2009, 2011, 2012 Broad Institute.
Author: Heng Li <lh3@sanger.ac.uk>
#include <unistd.h>
#include <math.h>
#include <inttypes.h>
-#include <stdbool.h>
-#include <assert.h>
#include <getopt.h>
-#include <ctype.h>
#include "htslib/sam.h"
#include "htslib/faidx.h"
-#include "htslib/kstring.h"
#include "htslib/khash.h"
-#include "htslib/klist.h"
#include "htslib/thread_pool.h"
-#include "htslib/bgzf.h"
#include "samtools.h"
#include "sam_opts.h"
#include "bedidx.h"
-#define DEFAULT_BARCODE_TAG "BC"
-#define DEFAULT_QUALITY_TAG "QT"
-
KHASH_SET_INIT_STR(rg)
-#define taglist_free(p)
-KLIST_INIT(ktaglist, char*, taglist_free)
+KHASH_SET_INIT_STR(tv)
typedef khash_t(rg) *rghash_t;
+typedef khash_t(tv) *tvhash_t;
// This structure contains the settings for a samview run
typedef struct samview_settings {
rghash_t rghash;
+ tvhash_t tvhash;
int min_mapQ;
int flag_on;
int flag_off;
size_t remove_aux_len;
char** remove_aux;
int multi_region;
+ char* tag;
} samview_settings_t;
// TODO Add declarations of these to a viable htslib or samtools header
-extern const char *bam_get_library(bam_hdr_t *header, const bam1_t *b);
+extern const char *bam_get_library(sam_hdr_t *header, const bam1_t *b);
extern int bam_remove_B(bam1_t *b);
extern char *samfaipath(const char *fn_ref);
// Returns 0 to indicate read should be output 1 otherwise
-static int process_aln(const bam_hdr_t *h, bam1_t *b, samview_settings_t* settings)
+static int process_aln(const sam_hdr_t *h, bam1_t *b, samview_settings_t* settings)
{
if (settings->remove_B) bam_remove_B(b);
if (settings->min_qlen > 0) {
return 1;
if (settings->flag_alloff && ((b->core.flag & settings->flag_alloff) == settings->flag_alloff))
return 1;
- if (!settings->multi_region && settings->bed && (b->core.tid < 0 || !bed_overlap(settings->bed, h->target_name[b->core.tid], b->core.pos, bam_endpos(b))))
+ if (!settings->multi_region && settings->bed && (b->core.tid < 0 || !bed_overlap(settings->bed, sam_hdr_tid2name(h, b->core.tid), b->core.pos, bam_endpos(b))))
return 1;
if (settings->subsam_frac > 0.) {
uint32_t k = __ac_Wang_hash(__ac_X31_hash_string(bam_get_qname(b)) ^ settings->subsam_seed);
if (k == kh_end(settings->rghash)) return 1;
}
}
+ if (settings->tvhash && settings->tag) {
+ uint8_t *s = bam_aux_get(b, settings->tag);
+ if (s) {
+ khint_t k = kh_get(tv, settings->tvhash, (char*)(s + 1));
+ if (k == kh_end(settings->tvhash)) return 1;
+ } else {
+ return 1;
+ }
+ }
if (settings->library) {
- const char *p = bam_get_library((bam_hdr_t*)h, b);
+ const char *p = bam_get_library((sam_hdr_t*)h, b);
if (!p || strcmp(p, settings->library) != 0) return 1;
}
if (settings->remove_aux_len) {
return 0;
}
-static char *drop_rg(char *hdtxt, rghash_t h, int *len)
-{
- char *p = hdtxt, *q, *r, *s;
- kstring_t str;
- memset(&str, 0, sizeof(kstring_t));
- while (1) {
- int toprint = 0;
- q = strchr(p, '\n');
- if (q == 0) q = p + strlen(p);
- if (q - p < 3) break; // the line is too short; then stop
- if (strncmp(p, "@RG\t", 4) == 0) {
- int c;
- khint_t k;
- if ((r = strstr(p, "\tID:")) != 0) {
- r += 4;
- for (s = r; *s != '\0' && *s != '\n' && *s != '\t'; ++s);
- c = *s; *s = '\0';
- k = kh_get(rg, h, r);
- *s = c;
- if (k != kh_end(h)) toprint = 1;
- }
- } else toprint = 1;
- if (toprint) {
- kputsn(p, q - p, &str); kputc('\n', &str);
- }
- p = q + 1;
- }
- *len = str.l;
- return str.s;
-}
-
static int usage(FILE *fp, int exit_status, int is_long_help);
static int add_read_group_single(const char *subcmd, samview_settings_t *settings, char *name)
return (ret != -1) ? 0 : -1;
}
-static inline int check_sam_write1(samFile *fp, const bam_hdr_t *h, const bam1_t *b, const char *fname, int *retp)
+static int add_tag_value_single(const char *subcmd, samview_settings_t *settings, char *name)
{
- int r = sam_write1(fp, h, b);
- if (r >= 0) return r;
+ char *d = strdup(name);
+ int ret = 0;
- if (fname) print_error_errno("view", "writing to \"%s\" failed", fname);
- else print_error_errno("view", "writing to standard output failed");
+ if (d == NULL) goto err;
- *retp = EXIT_FAILURE;
- return r;
+ if (settings->tvhash == NULL) {
+ settings->tvhash = kh_init(tv);
+ if (settings->tvhash == NULL) goto err;
+ }
+
+ kh_put(tv, settings->tvhash, d, &ret);
+ if (ret == -1) goto err;
+ if (ret == 0) free(d); /* Duplicate */
+ return 0;
+
+ err:
+ print_error(subcmd, "Couldn't add \"%s\" to tag values list: memory exhausted?", name);
+ free(d);
+ return -1;
}
-static void check_sam_close(const char *subcmd, samFile *fp, const char *fname, const char *null_fname, int *retp)
+static int add_tag_values_file(const char *subcmd, samview_settings_t *settings, char *fn)
{
- int r = sam_close(fp);
- if (r >= 0) return;
+ FILE *fp;
+ char buf[1024];
+ int ret = 0;
+ if (settings->tvhash == NULL) {
+ settings->tvhash = kh_init(tv);
+ if (settings->tvhash == NULL) {
+ perror(NULL);
+ return -1;
+ }
+ }
- // TODO Need error infrastructure so we can print a message instead of r
- if (fname) print_error(subcmd, "error closing \"%s\": %d", fname, r);
- else print_error(subcmd, "error closing %s: %d", null_fname, r);
+ fp = fopen(fn, "r");
+ if (fp == NULL) {
+ print_error_errno(subcmd, "failed to open \"%s\" for reading", fn);
+ return -1;
+ }
+
+ while (ret != -1 && !feof(fp) && fscanf(fp, "%1023s", buf) > 0) {
+ char *d = strdup(buf);
+ if (d != NULL) {
+ kh_put(tv, settings->tvhash, d, &ret);
+ if (ret == 0) free(d); /* Duplicate */
+ } else {
+ ret = -1;
+ }
+ }
+ if (ferror(fp)) ret = -1;
+ if (ret == -1) {
+ print_error_errno(subcmd, "failed to read \"%s\"", fn);
+ }
+ fclose(fp);
+ return (ret != -1) ? 0 : -1;
+}
+
+static inline int check_sam_write1(samFile *fp, const sam_hdr_t *h, const bam1_t *b, const char *fname, int *retp)
+{
+ int r = sam_write1(fp, h, b);
+ if (r >= 0) return r;
+
+ if (fname) print_error_errno("view", "writing to \"%s\" failed", fname);
+ else print_error_errno("view", "writing to standard output failed");
*retp = EXIT_FAILURE;
+ return r;
}
int main_samview(int argc, char *argv[])
{
- int c, is_header = 0, is_header_only = 0, ret = 0, compress_level = -1, is_count = 0;
+ int c, is_header = 0, is_header_only = 0, ret = 0, compress_level = -1, is_count = 0, has_index_file = 0, no_pg = 0;
int64_t count = 0;
samFile *in = 0, *out = 0, *un_out=0;
FILE *fp_out = NULL;
- bam_hdr_t *header = NULL;
+ sam_hdr_t *header = NULL;
char out_mode[5], out_un_mode[5], *out_format = "";
- char *fn_in = 0, *fn_out = 0, *fn_list = 0, *q, *fn_un_out = 0;
+ char *fn_in = 0, *fn_idx_in = 0, *fn_out = 0, *fn_list = 0, *q, *fn_un_out = 0;
+ char *fn_out_idx = NULL, *fn_un_out_idx = NULL, *arg_list = NULL;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
htsThreadPool p = {NULL, 0};
int filter_state = ALL, filter_op = 0;
samview_settings_t settings = {
.rghash = NULL,
+ .tvhash = NULL,
.min_mapQ = 0,
.flag_on = 0,
.flag_off = 0,
.subsam_frac = -1.,
.library = NULL,
.bed = NULL,
- .multi_region = 0
+ .multi_region = 0,
+ .tag = NULL
};
static const struct option lopts[] = {
SAM_OPT_GLOBAL_OPTIONS('-', 0, 'O', 0, 'T', '@'),
+ {"no-PG", no_argument, NULL, 1},
{ NULL, 0, NULL, 0 }
};
opterr = 0;
while ((c = getopt_long(argc, argv,
- "SbBcCt:h1Ho:O:q:f:F:G:ul:r:T:R:L:s:@:m:x:U:M",
+ "SbBcCt:h1Ho:O:q:f:F:G:ul:r:T:R:d:D:L:s:@:m:x:U:MX",
lopts, NULL)) >= 0) {
switch (c) {
case 's':
srand(settings.subsam_seed);
settings.subsam_seed = rand();
}
-
if (q && *q == '.') {
settings.subsam_frac = strtod(q, &q);
if (*q) ret = 1;
case 'H': is_header_only = 1; break;
case 'o': fn_out = strdup(optarg); break;
case 'U': fn_un_out = strdup(optarg); break;
+ case 'X': has_index_file = 1; break;
case 'f': settings.flag_on |= strtol(optarg, 0, 0); break;
case 'F': settings.flag_off |= strtol(optarg, 0, 0); break;
case 'G': settings.flag_alloff |= strtol(optarg, 0, 0); break;
ret = 1;
goto view_end;
}
+ break;
+ case 'd':
+ if (strlen(optarg) < 4 || optarg[2] != ':') {
+ print_error_errno("view", "Invalid \"tag:value\" option: \"%s\"", optarg);
+ ret = 1;
+ goto view_end;
+ }
+
+ if (settings.tag) {
+ if (settings.tag[0] != optarg[0] || settings.tag[1] != optarg[1]) {
+ print_error("view", "Different tag \"%s\" was specified before: \"%s\"", settings.tag, optarg);
+ ret = 1;
+ goto view_end;
+ }
+ } else {
+ if (!(settings.tag = calloc(3, 1))) {
+ print_error("view", "Could not allocate memory for tag: \"%s\"", optarg);
+ ret = 1;
+ goto view_end;
+ }
+ memcpy(settings.tag, optarg, 2);
+ }
+
+ if (add_tag_value_single("view", &settings, optarg+3) != 0) {
+ ret = 1;
+ goto view_end;
+ }
+ break;
+ case 'D':
+ // Allow ";" as delimiter besides ":" to support MinGW CLI POSIX
+ // path translation as described at:
+ // http://www.mingw.org/wiki/Posix_path_conversion
+ if (strlen(optarg) < 4 || (optarg[2] != ':' && optarg[2] != ';')) {
+ print_error_errno("view", "Invalid \"tag:file\" option: \"%s\"", optarg);
+ ret = 1;
+ goto view_end;
+ }
+
+ if (settings.tag) {
+ if (settings.tag[0] != optarg[0] || settings.tag[1] != optarg[1]) {
+ print_error("view", "Different tag \"%s\" was specified before: \"%s\"", settings.tag, optarg);
+ ret = 1;
+ goto view_end;
+ }
+ } else {
+ if (!(settings.tag = calloc(3, 1))) {
+ print_error("view", "Could not allocate memory for tag: \"%s\"", optarg);
+ ret = 1;
+ goto view_end;
+ }
+ memcpy(settings.tag, optarg, 2);
+ }
+
+ if (add_tag_values_file("view", &settings, optarg+3) != 0) {
+ ret = 1;
+ goto view_end;
+ }
break;
/* REMOVED as htslib doesn't support this
//case 'x': out_format = "x"; break;
}
break;
case 'M': settings.multi_region = 1; break;
+ case 1: no_pg = 1; break;
default:
if (parse_sam_global_opt(c, optarg, lopts, &ga) != 0)
return usage(samtools_stderr, EXIT_FAILURE, 0);
ret = 1;
goto view_end;
}
- if (settings.rghash) { // FIXME: I do not know what "bam_header_t::n_text" is for...
- char *tmp;
- int l;
- tmp = drop_rg(header->text, settings.rghash, &l);
- free(header->text);
- header->text = tmp;
- header->l_text = l;
+ if (settings.rghash) {
+ sam_hdr_remove_lines(header, "RG", "ID", settings.rghash);
}
if (!is_count) {
if ((out = sam_open_format(fn_out? fn_out : "-", out_mode, &ga.out)) == 0) {
goto view_end;
}
}
- if (*out_format || is_header ||
+
+ if (!no_pg) {
+ if (!(arg_list = stringify_argv(argc+1, argv-1))) {
+ print_error("view", "failed to create arg_list");
+ ret = 1;
+ goto view_end;
+ }
+ if (sam_hdr_add_pg(header, "samtools",
+ "VN", samtools_version(),
+ arg_list ? "CL": NULL,
+ arg_list ? arg_list : NULL,
+ NULL)) {
+ print_error("view", "failed to add PG line to the header");
+ ret = 1;
+ goto view_end;
+ }
+ }
+
+ if (*out_format || ga.write_index || is_header ||
out_mode[1] == 'b' || out_mode[1] == 'c' ||
(ga.out.format != sam && ga.out.format != unknown_format)) {
if (sam_hdr_write(out, header) != 0) {
goto view_end;
}
}
+ if (ga.write_index) {
+ if (!(fn_out_idx = auto_index(out, fn_out, header))) {
+ ret = 1;
+ goto view_end;
+ }
+ }
+
if (fn_un_out) {
if ((un_out = sam_open_format(fn_un_out, out_un_mode, &ga.out)) == 0) {
print_error_errno("view", "failed to open \"%s\" for writing", fn_un_out);
goto view_end;
}
}
+ if (ga.write_index) {
+ if (!(fn_un_out_idx = auto_index(un_out, fn_un_out, header))) {
+ ret = 1;
+ goto view_end;
+ }
+ }
}
}
else {
}
if (is_header_only) goto view_end; // no need to print alignments
+ if (has_index_file) {
+ fn_idx_in = (optind+1 < argc)? argv[optind+1] : 0;
+ if (fn_idx_in == 0) {
+ fprintf(samtools_stderr, "[main_samview] incorrect number of arguments for -X option. Aborting.\n");
+ return 1;
+ }
+ }
+
if (settings.multi_region) {
- if (optind < argc - 1) { //regions have been specified in the command line
+ if (!has_index_file && optind < argc - 1) { //regions have been specified in the command line
settings.bed = bed_hash_regions(settings.bed, argv, optind+1, argc, &filter_op); //insert(1) or filter out(0) the regions from the command line in the same hash table as the bed file
if (!filter_op)
filter_state = FILTERED;
+ } else if (has_index_file && optind < argc - 2) {
+ settings.bed = bed_hash_regions(settings.bed, argv, optind+2, argc, &filter_op); //insert(1) or filter out(0) the regions from the command line in the same hash table as the bed file
+ if (!filter_op)
+ filter_state = FILTERED;
} else {
bed_unify(settings.bed);
}
if (settings.bed == NULL) { // index is unavailable or no regions have been specified
fprintf(samtools_stderr, "[main_samview] no regions or BED file have been provided. Aborting.\n");
} else {
- hts_idx_t *idx = sam_index_load(in, fn_in); // load index
+ hts_idx_t *idx = NULL;
+ // If index filename has not been specfied, look in BAM folder
+ if (fn_idx_in != 0) {
+ idx = sam_index_load2(in, fn_in, fn_idx_in); // load index
+ } else {
+ idx = sam_index_load(in, fn_in);
+ }
if (idx != NULL) {
int regcount = 0;
}
bam_destroy1(b);
} else {
- if (optind + 1 >= argc) { // convert/print the entire file
+ if ((has_index_file && optind >= argc - 2) || (!has_index_file && optind >= argc - 1)) { // convert/print the entire file
bam1_t *b = bam_init1();
int r;
while ((r = sam_read1(in, header, b)) >= 0) { // read one alignment from `in'
} else { // retrieve alignments in specified regions
int i;
bam1_t *b;
- hts_idx_t *idx = sam_index_load(in, fn_in); // load index
+ hts_idx_t *idx = NULL;
+ // If index filename has not been specfied, look in BAM folder
+ if (fn_idx_in != NULL) {
+ idx = sam_index_load2(in, fn_in, fn_idx_in); // load index
+ } else {
+ idx = sam_index_load(in, fn_in);
+ }
if (idx == 0) { // index is unavailable
fprintf(samtools_stderr, "[main_samview] random alignment retrieval only works for indexed BAM or CRAM files.\n");
ret = 1;
goto view_end;
}
b = bam_init1();
- for (i = optind + 1; i < argc; ++i) {
+
+ for (i = (has_index_file)? optind+2 : optind+1; i < argc; ++i) {
int result;
hts_itr_t *iter = sam_itr_querys(idx, header, argv[i]); // parse a region in the format like `chr2:100-200'
if (iter == NULL) { // region invalid or reference name not found
- int beg, end;
- if (hts_parse_reg(argv[i], &beg, &end))
- fprintf(samtools_stderr, "[main_samview] region \"%s\" specifies an unknown reference name. Continue anyway.\n", argv[i]);
- else
- fprintf(samtools_stderr, "[main_samview] region \"%s\" could not be parsed. Continue anyway.\n", argv[i]);
+ fprintf(samtools_stderr, "[main_samview] region \"%s\" specifies an invalid region or unknown reference. Continue anyway.\n", argv[i]);
continue;
}
// fetch alignments
}
}
+ if (ga.write_index) {
+ if (sam_idx_save(out) < 0) {
+ print_error_errno("view", "writing index failed");
+ ret = 1;
+ }
+ if (un_out && sam_idx_save(un_out) < 0) {
+ print_error_errno("view", "writing index failed");
+ ret = 1;
+ }
+ }
+
view_end:
if (is_count && ret == 0) {
if (fprintf(fn_out? fp_out : samtools_stdout, "%" PRId64 "\n", count) < 0) {
free(fn_list); free(fn_out); free(settings.library); free(fn_un_out);
sam_global_args_free(&ga);
- if ( header ) bam_hdr_destroy(header);
+ if ( header ) sam_hdr_destroy(header);
if (settings.bed) bed_destroy(settings.bed);
if (settings.rghash) {
khint_t k;
if (kh_exist(settings.rghash, k)) free((char*)kh_key(settings.rghash, k));
kh_destroy(rg, settings.rghash);
}
+ if (settings.tvhash) {
+ khint_t k;
+ for (k = 0; k < kh_end(settings.tvhash); ++k)
+ if (kh_exist(settings.tvhash, k)) free((char*)kh_key(settings.tvhash, k));
+ kh_destroy(tv, settings.tvhash);
+ }
if (settings.remove_aux_len) {
free(settings.remove_aux);
}
+ if (settings.tag) {
+ free(settings.tag);
+ }
if (p.pool)
hts_tpool_destroy(p.pool);
+ if (fn_out_idx)
+ free(fn_out_idx);
+ if (fn_un_out_idx)
+ free(fn_un_out_idx);
+ free(arg_list);
+
return ret;
}
" -U FILE output reads not selected by filters to FILE [null]\n"
// extra input
" -t FILE FILE listing reference names and lengths (see long help) [null]\n"
+" -X include customized index file\n"
// read filters
" -L FILE only include reads overlapping this BED FILE [null]\n"
" -r STR only include reads in read group STR [null]\n"
" -R FILE only include reads with read group listed in FILE [null]\n"
+" -d STR:STR\n"
+" only include reads with tag STR and associated value STR [null]\n"
+" -D STR:FILE\n"
+" only include reads with tag STR and associated values listed in\n"
+" FILE [null]\n"
" -q INT only include reads with mapping quality >= INT [0]\n"
" -l STR only include reads in library STR [null]\n"
" -m INT only include reads with number of CIGAR operations consuming\n"
" -B collapse the backward CIGAR operation\n"
// general options
" -? print long help, including note about region specification\n"
-" -S ignored (input format is auto-detected)\n");
+" -S ignored (input format is auto-detected)\n"
+" --no-PG do not add a PG line\n");
- sam_global_opt_help(fp, "-.O.T@");
+ sam_global_opt_help(fp, "-.O.T@..");
fprintf(fp, "\n");
if (is_long_help)
free(argv2);
return ret;
}
-
-int8_t seq_comp_table[16] = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 };
-static const char *copied_tags[] = { "RG", "BC", "QT", NULL };
-
-static void bam2fq_usage(FILE *to, const char *command)
-{
- int fq = strcasecmp("fastq", command) == 0 || strcasecmp("bam2fq", command) == 0;
- fprintf(to,
-"Usage: samtools %s [options...] <in.bam>\n", command);
- fprintf(to,
-"Options:\n"
-" -0 FILE write reads designated READ_OTHER to FILE\n"
-" -1 FILE write reads designated READ1 to FILE\n"
-" -2 FILE write reads designated READ2 to FILE\n"
-" note: if a singleton file is specified with -s, only\n"
-" paired reads will be written to the -1 and -2 files.\n"
-" -f INT only include reads with all of the FLAGs in INT present [0]\n" // F&x == x
-" -F INT only include reads with none of the FLAGS in INT present [0]\n" // F&x == 0
-" -G INT only EXCLUDE reads with all of the FLAGs in INT present [0]\n" // !(F&x == x)
-" -n don't append /1 and /2 to the read name\n"
-" -N always append /1 and /2 to the read name\n");
- if (fq) fprintf(to,
-" -O output quality in the OQ tag if present\n");
- fprintf(to,
-" -s FILE write singleton reads designated READ1 or READ2 to FILE\n"
-" -t copy RG, BC and QT tags to the %s header line\n",
- fq ? "FASTQ" : "FASTA");
- fprintf(to,
-" -T TAGLIST copy arbitrary tags to the %s header line\n",
- fq ? "FASTQ" : "FASTA");
- if (fq) fprintf(to,
-" -v INT default quality score if not given in file [1]\n"
-" -i add Illumina Casava 1.8 format entry to header (eg 1:N:0:ATCACG)\n"
-" -c compression level [0..9] to use when creating gz or bgzf fastq files\n"
-" --i1 FILE write first index reads to FILE\n"
-" --i2 FILE write second index reads to FILE\n"
-" --barcode-tag TAG Barcode tag [default: " DEFAULT_BARCODE_TAG "]\n"
-" --quality-tag TAG Quality tag [default: " DEFAULT_QUALITY_TAG "]\n"
-" --index-format STR How to parse barcode and quality tags\n\n");
- sam_global_opt_help(to, "-.--.@");
- fprintf(to,
-"\n"
-"Reads are designated READ1 if FLAG READ1 is set and READ2 is not set.\n"
-"Reads are designated READ2 if FLAG READ1 is not set and READ2 is set.\n"
-"Reads are designated READ_OTHER if FLAGs READ1 and READ2 are either both set\n"
-"or both unset.\n"
-"Run 'samtools flags' for more information on flag codes and meanings.\n");
- fprintf(to,
-"\n"
-"The index-format string describes how to parse the barcode and quality tags, for example:\n"
-" i14i8 the first 14 characters are index 1, the next 8 characters are index 2\n"
-" n8i14 ignore the first 8 characters, and use the next 14 characters for index 1\n"
-"If the tag contains a separator, then the numeric part can be replaced with '*' to mean\n"
-"'read until the separator or end of tag', for example:\n"
-" n*i* ignore the left part of the tag until the separator, then use the second part\n"
-" of the tag as index 1\n");
- fprintf(to,
-"\n"
-"Examples:\n"
-" To get just the paired reads in separate files, use:\n"
-" samtools %s -1 paired1.%s -2 paired2.%s -0 /dev/null -s /dev/null -n -F 0x900 in.bam\n"
-"\n To get all non-supplementary/secondary reads in a single file, redirect the output:\n"
-" samtools %s -F 0x900 in.bam > all_reads.%s\n",
- command, fq ? "fq" : "fa", fq ? "fq" : "fa",
- command, fq ? "fq" : "fa");
-}
-
-typedef enum { READ_UNKNOWN = 0, READ_1 = 1, READ_2 = 2 } readpart;
-typedef enum { FASTA, FASTQ } fastfile;
-typedef struct bam2fq_opts {
- char *fnse;
- char *fnr[3];
- char *fn_input; // pointer to input filename in argv do not free
- bool has12, has12always, use_oq, copy_tags, illumina_tag;
- int flag_on, flag_off, flag_alloff;
- sam_global_args ga;
- fastfile filetype;
- int def_qual;
- char *barcode_tag;
- char *quality_tag;
- char *index_file[2];
- char *index_format;
- char *extra_tags;
- char compression_level;
-} bam2fq_opts_t;
-
-typedef struct bam2fq_state {
- samFile *fp;
- BGZF *fpse;
- BGZF *fpr[3];
- BGZF *fpi[2];
- BGZF *hsamtools_stdout;
- bam_hdr_t *h;
- bool has12, use_oq, copy_tags, illumina_tag;
- int flag_on, flag_off, flag_alloff;
- fastfile filetype;
- int def_qual;
- klist_t(ktaglist) *taglist;
- char *index_sequence;
- char compression_level;
-} bam2fq_state_t;
-
-/*
- * Get and decode the read from a BAM record.
- *
- * TODO: htslib really needs an interface for this. Consider this or perhaps
- * bam_get_seq_str (current vs original orientation) and bam_get_qual_str
- * functions as string formatted equivalents to bam_get_{seq,qual}?
- */
-
-/*
- * Reverse a string in place.
- * From http://stackoverflow.com/questions/8534274/is-the-strrev-function-not-available-in-linux.
- * Author Sumit-naik: http://stackoverflow.com/users/4590926/sumit-naik
- */
-static char *reverse(char *str)
-{
- int i = strlen(str)-1,j=0;
- char ch;
- while (i>j) {
- ch = str[i];
- str[i]= str[j];
- str[j] = ch;
- i--;
- j++;
- }
- return str;
-}
-
-/* return the read, reverse complemented if necessary */
-static char *get_read(const bam1_t *rec)
-{
- int len = rec->core.l_qseq + 1;
- char *read = calloc(1, len);
- char *seq = (char *)bam_get_seq(rec);
- int n;
-
- if (!read) return NULL;
-
- for (n=0; n < rec->core.l_qseq; n++) {
- if (rec->core.flag & BAM_FREVERSE) read[n] = seq_nt16_str[seq_comp_table[bam_seqi(seq,n)]];
- else read[n] = seq_nt16_str[bam_seqi(seq,n)];
- }
- if (rec->core.flag & BAM_FREVERSE) reverse(read);
- return read;
-}
-
-/*
- * get and decode the quality from a BAM record
- */
-static int get_quality(const bam1_t *rec, char **qual_out)
-{
- char *quality = calloc(1, rec->core.l_qseq + 1);
- char *q = (char *)bam_get_qual(rec);
- int n;
-
- if (!quality) return -1;
-
- if (*q == '\xff') {
- free(quality);
- *qual_out = NULL;
- return 0;
- }
-
- for (n=0; n < rec->core.l_qseq; n++) {
- quality[n] = q[n]+33;
- }
- if (rec->core.flag & BAM_FREVERSE) reverse(quality);
- *qual_out = quality;
- return 0;
-}
-
-//
-// End of htslib complaints
-//
-
-
-static readpart which_readpart(const bam1_t *b)
-{
- if ((b->core.flag & BAM_FREAD1) && !(b->core.flag & BAM_FREAD2)) {
- return READ_1;
- } else if ((b->core.flag & BAM_FREAD2) && !(b->core.flag & BAM_FREAD1)) {
- return READ_2;
- } else {
- return READ_UNKNOWN;
- }
-}
-
-/*
- * parse the length part from the index-format string
- */
-static int getLength(char **s)
-{
- int n = 0;
- while (**s) {
- if (**s == '*') { n=-1; (*s)++; break; }
- if ( !isdigit(**s)) break;
- n = n*10 + ((**s)-'0');
- (*s)++;
- }
- return n;
-}
-
-static bool copy_tag(const char *tag, const bam1_t *rec, kstring_t *linebuf)
-{
- uint8_t *s = bam_aux_get(rec, tag);
- if (s) {
- char aux_type = *s;
- switch (aux_type) {
- case 'C':
- case 'S': aux_type = 'I'; break;
- case 'c':
- case 's': aux_type = 'i'; break;
- case 'd': aux_type = 'f'; break;
- }
-
- // Ensure space. Need 6 chars + length of tag. Max length of
- // i is 16, A is 21, B currently 26, Z is unknown, so
- // have to check that one later.
- if (ks_resize(linebuf, ks_len(linebuf) + 64) < 0) return false;
-
- kputc('\t', linebuf);
- kputsn(tag, 2, linebuf);
- kputc(':', linebuf);
- kputc(aux_type=='I'? 'i': aux_type, linebuf);
- kputc(':', linebuf);
- switch (aux_type) {
- case 'H':
- case 'Z':
- if (kputs(bam_aux2Z(s), linebuf) < 0) return false;
- break;
- case 'i': kputw(bam_aux2i(s), linebuf); break;
- case 'I': kputuw(bam_aux2i(s), linebuf); break;
- case 'A': kputc(bam_aux2A(s), linebuf); break;
- case 'f': kputd(bam_aux2f(s), linebuf); break;
- case 'B': kputs("*** Unhandled aux type ***", linebuf); return false;
- default: kputs("*** Unknown aux type ***", linebuf); return false;
- }
- }
- return true;
-}
-
-static int insert_index_sequence_into_linebuf(char *index_sequence, kstring_t *linebuf, bam1_t *rec)
-{
- if (!index_sequence) return 0;
-
- kstring_t new = {0,0,NULL};
- if (linebuf->s) {
- char *s = strchr(linebuf->s, '\n');
- if (s) {
- if (ks_resize(&new, linebuf->l + strlen(index_sequence) + 16) < 0)
- return -1;
- *s = 0;
- kputs(linebuf->s, &new);
- kputc(' ', &new);
- readpart readpart = which_readpart(rec);
- if (readpart == READ_1) kputc('1', &new);
- else if (readpart == READ_2) kputc('2', &new);
- else kputc('0', &new);
-
- kputc(':', &new);
- if (rec->core.flag & BAM_FQCFAIL) kputc('Y', &new);
- else kputc('N', &new);
-
- kputs(":0:", &new);
- kputs(index_sequence, &new);
- kputc('\n', &new);
- kputs(s+1, &new);
- free(ks_release(linebuf));
- linebuf->s = new.s; linebuf->l = new.l; linebuf->m = new.m;
- }
- }
- return 0;
-}
-
-static bool make_fq_line(const bam1_t *rec, char *seq, char *qual, kstring_t *linebuf, const bam2fq_state_t *state)
-{
- int i;
-
- linebuf->l = 0;
- // Write read name
- if (kputc(state->filetype == FASTA? '>' : '@', linebuf) < 0) return false;
- if (kputs(bam_get_qname(rec), linebuf) < 0) return false;
- // Add the /1 /2 if requested
- if (state->has12) {
- readpart readpart = which_readpart(rec);
- if (readpart == READ_1) {
- if (kputs("/1", linebuf) < 0) return false;
- } else if (readpart == READ_2) {
- if (kputs("/2", linebuf) < 0) return false;
- }
- }
- if (state->copy_tags) {
- for (i = 0; copied_tags[i]; ++i) {
- if (!copy_tag(copied_tags[i], rec, linebuf)) {
- fprintf(samtools_stderr, "Problem copying aux tags: [%s]\n", linebuf->s);
- return false;
- }
- }
- }
-
- if (state->taglist->size) {
- kliter_t(ktaglist) *p;
- for (p = kl_begin(state->taglist); p != kl_end(state->taglist); p = kl_next(p)) {
- if (!copy_tag(kl_val(p), rec, linebuf)) {
- fprintf(samtools_stderr, "Problem copying aux tags: [%s]\n", linebuf->s);
- return false;
- }
- }
- }
-
- if (kputc('\n', linebuf) < 0) return false;
- if (kputs(seq, linebuf) < 0) return false;
- if (kputc('\n', linebuf) < 0) return false;
-
- if (state->filetype == FASTQ) {
- // Write quality
- if (kputs("+\n", linebuf) < 0) return false;
- if (qual && *qual) {
- if (kputs(qual, linebuf) < 0) return false;
- } else {
- int len = strlen(seq);
- if (ks_resize(linebuf, ks_len(linebuf) + len + 1) < 0) return false;
- for (i = 0; i < len; ++i) {
- kputc(33 + state->def_qual, linebuf);
- }
- }
- if (kputc('\n', linebuf) < 0) return false;
- }
- return true;
-}
-
-/*
- * Create FASTQ lines from the barcode tag using the index-format
- */
-static bool tags2fq(bam1_t *rec, bam2fq_state_t *state, const bam2fq_opts_t* opts)
-{
- uint8_t *p;
- char *ifmt = opts->index_format;
- char *tag = NULL;
- char *qual = NULL;
- char *sub_tag = NULL;
- char *sub_qual = NULL;
- size_t tag_len;
- int file_number = 0;
- kstring_t linebuf = { 0, 0, NULL }; // Buffer
-
-
- // read barcode tag
- p = bam_aux_get(rec,opts->barcode_tag);
- if (p) tag = bam_aux2Z(p);
-
- if (!tag) return true; // there is no tag
-
- tag_len = strlen(tag);
- sub_tag = calloc(1, tag_len + 1);
- if (!sub_tag) goto fail;
- sub_qual = calloc(1, tag_len + 1);
- if (!sub_qual) goto fail;
-
- // read quality tag
- p = bam_aux_get(rec, opts->quality_tag);
- if (p) qual = bam_aux2Z(p);
-
- // Parse the index-format string
- while (*ifmt) {
- if (file_number > 1) break; // shouldn't happen if we've validated paramaters correctly
- char action = *ifmt; // should be 'i' or 'n'
- ifmt++; // skip over action
- int index_len = getLength(&ifmt);
- int n = 0;
-
- if (index_len < 0) {
- // read until separator
- while (isalpha(*tag)) {
- sub_tag[n] = *tag++;
- if (qual) sub_qual[n] = *qual++;
- n++;
- }
- if (*tag) { // skip separator
- tag++;
- if (qual) qual++;
- }
- } else {
- // read index_len characters
- while (index_len-- && *tag) {
- sub_tag[n] = *tag++;
- if (qual) sub_qual[n] = *qual++;
- n++;
- }
- }
- sub_tag[n] = '\0';
- sub_qual[n] = '\0';
-
- if (action=='i' && *sub_tag && state->fpi[file_number]) {
- //if (file_number==0) state->index_sequence = strdup(sub_tag); // we're going to need this later...
- state->index_sequence = strdup(sub_tag); // we're going to need this later...
- if (!state->index_sequence) goto fail;
- if (!make_fq_line(rec, sub_tag, sub_qual, &linebuf, state)) goto fail;
- if (state->illumina_tag) {
- if (insert_index_sequence_into_linebuf(state->index_sequence, &linebuf, rec) < 0) {
- goto fail;
- }
- }
- if (bgzf_write(state->fpi[file_number++], linebuf.s, linebuf.l) < 0)
- goto fail;
- }
-
- }
-
- free(sub_qual); free(sub_tag);
- free(linebuf.s);
- return true;
-
- fail:
- perror(__func__);
- free(sub_qual); free(sub_tag);
- free(linebuf.s);
- return true;
-}
-
-// Transform a bam1_t record into a string with the FASTQ representation of it
-// @returns false for error, true for success
-static bool bam1_to_fq(const bam1_t *b, kstring_t *linebuf, const bam2fq_state_t *state)
-{
- int32_t qlen = b->core.l_qseq;
- assert(qlen >= 0);
- const uint8_t *oq = NULL;
- char *qual = NULL;
-
- char *seq = get_read(b);
- if (!seq) return false;
-
- if (state->use_oq) oq = bam_aux_get(b, "OQ");
- if (oq && *oq=='Z') {
- qual = strdup(bam_aux2Z(oq));
- if (!qual) goto fail;
- if (b->core.flag & BAM_FREVERSE) { // read is reverse complemented
- reverse(qual);
- }
- } else {
- if (get_quality(b, &qual) < 0) goto fail;
- }
-
- if (!make_fq_line(b, seq, qual, linebuf, state)) goto fail;
-
- free(qual);
- free(seq);
- return true;
-
- fail:
- free(seq);
- free(qual);
- return false;
-}
-
-static void free_opts(bam2fq_opts_t *opts)
-{
- free(opts->barcode_tag);
- free(opts->quality_tag);
- free(opts->index_format);
- free(opts->extra_tags);
- free(opts);
-}
-
-// return true if valid
-static bool parse_opts(int argc, char *argv[], bam2fq_opts_t** opts_out)
-{
- // Parse args
- bam2fq_opts_t* opts = calloc(1, sizeof(bam2fq_opts_t));
- opts->has12 = true;
- opts->has12always = false;
- opts->filetype = FASTQ;
- opts->def_qual = 1;
- opts->barcode_tag = NULL;
- opts->quality_tag = NULL;
- opts->index_format = NULL;
- opts->index_file[0] = NULL;
- opts->index_file[1] = NULL;
- opts->extra_tags = NULL;
- opts->compression_level = 1;
-
- int c;
- sam_global_args_init(&opts->ga);
- static const struct option lopts[] = {
- SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0, '@'),
- {"i1", required_argument, NULL, 1},
- {"I1", required_argument, NULL, 1},
- {"i2", required_argument, NULL, 2},
- {"I2", required_argument, NULL, 2},
- {"if", required_argument, NULL, 3},
- {"IF", required_argument, NULL, 3},
- {"index-format", required_argument, NULL, 3},
- {"barcode-tag", required_argument, NULL, 'b'},
- {"quality-tag", required_argument, NULL, 'q'},
- { NULL, 0, NULL, 0 }
- };
- while ((c = getopt_long(argc, argv, "0:1:2:f:F:G:niNOs:c:tT:v:@:", lopts, NULL)) > 0) {
- switch (c) {
- case 'b': opts->barcode_tag = strdup(optarg); break;
- case 'q': opts->quality_tag = strdup(optarg); break;
- case 1 : opts->index_file[0] = optarg; break;
- case 2 : opts->index_file[1] = optarg; break;
- case 3 : opts->index_format = strdup(optarg); break;
- case '0': opts->fnr[0] = optarg; break;
- case '1': opts->fnr[1] = optarg; break;
- case '2': opts->fnr[2] = optarg; break;
- case 'f': opts->flag_on |= strtol(optarg, 0, 0); break;
- case 'F': opts->flag_off |= strtol(optarg, 0, 0); break;
- case 'G': opts->flag_alloff |= strtol(optarg, 0, 0); break;
- case 'n': opts->has12 = false; break;
- case 'N': opts->has12always = true; break;
- case 'O': opts->use_oq = true; break;
- case 's': opts->fnse = optarg; break;
- case 't': opts->copy_tags = true; break;
- case 'i': opts->illumina_tag = true; break;
- case 'c': opts->compression_level = atoi(optarg); break;
- case 'T': opts->extra_tags = strdup(optarg); break;
- case 'v': opts->def_qual = atoi(optarg); break;
- case '?': bam2fq_usage(samtools_stderr, argv[0]); free_opts(opts); return false;
- default:
- if (parse_sam_global_opt(c, optarg, lopts, &opts->ga) != 0) {
- bam2fq_usage(samtools_stderr, argv[0]); free_opts(opts); return false;
- }
- break;
- }
- }
-
- if (opts->fnr[1] || opts->fnr[2]) opts->has12 = false;
- if (opts->has12always) opts->has12 = true;
-
- if (!opts->barcode_tag) opts->barcode_tag = strdup(DEFAULT_BARCODE_TAG);
- if (!opts->quality_tag) opts->quality_tag = strdup(DEFAULT_QUALITY_TAG);
-
- int nIndex = 0;
- if (opts->index_format) {
- char *s;
- for (s = opts->index_format; *s; s++) {
- if (*s == 'i') nIndex++;
- }
- }
- if (nIndex>2) {
- fprintf(samtools_stderr,"Invalid index format: more than 2 indexes\n");
- bam2fq_usage(samtools_stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if (opts->index_file[1] && !opts->index_file[0]) {
- fprintf(samtools_stderr, "Index one specified, but index two not given\n");
- bam2fq_usage(samtools_stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if (nIndex==2 && !opts->index_file[1]) {
- fprintf(samtools_stderr, "index_format specifies two indexes, but only one index file given\n");
- bam2fq_usage(samtools_stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if (nIndex==1 && !opts->index_file[0]) {
- fprintf(samtools_stderr, "index_format specifies an index, but no index file given\n");
- bam2fq_usage(samtools_stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if (nIndex==0 && opts->index_file[0]) {
- fprintf(samtools_stderr, "index_format not specified, but index file given\n");
- bam2fq_usage(samtools_stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if (opts->def_qual < 0 || 93 < opts->def_qual) {
- fprintf(samtools_stderr, "Invalid -v default quality %i, allowed range 0 to 93\n", opts->def_qual);
- bam2fq_usage(samtools_stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- const char* type_str = argv[0];
- if (strcasecmp("fastq", type_str) == 0 || strcasecmp("bam2fq", type_str) == 0) {
- opts->filetype = FASTQ;
- } else if (strcasecmp("fasta", type_str) == 0) {
- opts->filetype = FASTA;
- } else {
- print_error("bam2fq", "Unrecognised type call \"%s\", this should be impossible... but you managed it!", type_str);
- bam2fq_usage(samtools_stderr, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if ((argc - (optind)) == 0) {
- fprintf(samtools_stderr, "No input file specified.\n");
- bam2fq_usage(samtools_stdout, argv[0]);
- free_opts(opts);
- return false;
- }
-
- if ((argc - (optind)) != 1) {
- fprintf(samtools_stderr, "Too many arguments.\n");
- bam2fq_usage(samtools_stderr, argv[0]);
- free_opts(opts);
- return false;
- }
- opts->fn_input = argv[optind];
- *opts_out = opts;
- return true;
-}
-
-static BGZF *open_fqfile(char *filename, int c)
-{
- char mode[4] = "w";
- size_t len = strlen(filename);
-
- mode[2] = 0; mode[3] = 0;
- if (len > 3 && strstr(filename + (len - 3),".gz")) {
- mode[1] = 'g'; mode[2] = c+'0';
- } else if ((len > 4 && strstr(filename + (len - 4),".bgz"))
- || (len > 5 && strstr(filename + (len - 5),".bgzf"))) {
- mode[1] = c+'0';
- } else {
- mode[1] = 'u';
- }
-
- return bgzf_open(filename,mode);
-}
-
-static bool init_state(const bam2fq_opts_t* opts, bam2fq_state_t** state_out)
-{
- bam2fq_state_t* state = calloc(1, sizeof(bam2fq_state_t));
- state->flag_on = opts->flag_on;
- state->flag_off = opts->flag_off;
- state->flag_alloff = opts->flag_alloff;
- state->has12 = opts->has12;
- state->use_oq = opts->use_oq;
- state->illumina_tag = opts->illumina_tag;
- state->copy_tags = opts->copy_tags;
- state->filetype = opts->filetype;
- state->def_qual = opts->def_qual;
- state->index_sequence = NULL;
- state->hsamtools_stdout = NULL;
- state->compression_level = opts->compression_level;
-
- state->taglist = kl_init(ktaglist);
- if (opts->extra_tags) {
- char *save_p;
- char *s = strtok_r(opts->extra_tags, ",", &save_p);
- while (s) {
- if (strlen(s) != 2) {
- fprintf(samtools_stderr, "Parsing extra tags - '%s' is not two characters\n", s);
- free(state);
- return false;
- }
- char **et = kl_pushp(ktaglist, state->taglist);
- *et = s;
- s = strtok_r(NULL, ",", &save_p);
- }
- }
-
- state->fp = sam_open(opts->fn_input, "r");
- if (state->fp == NULL) {
- print_error_errno("bam2fq","Cannot read file \"%s\"", opts->fn_input);
- free(state);
- return false;
- }
- if (opts->ga.nthreads > 0)
- hts_set_threads(state->fp, opts->ga.nthreads);
- uint32_t rf = SAM_QNAME | SAM_FLAG | SAM_SEQ | SAM_QUAL;
- if (opts->use_oq || opts->extra_tags || opts->index_file[0]) rf |= SAM_AUX;
- if (hts_set_opt(state->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
- fprintf(samtools_stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
- free(state);
- return false;
- }
- if (hts_set_opt(state->fp, CRAM_OPT_DECODE_MD, 0)) {
- fprintf(samtools_stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
- free(state);
- return false;
- }
- if (opts->fnse) {
- state->fpse = open_fqfile(opts->fnse, state->compression_level);
- if (state->fpse == NULL) {
- print_error_errno("bam2fq", "Cannot write to singleton file \"%s\"", opts->fnse);
- free(state);
- return false;
- }
- }
-
- if (opts->ga.reference) {
- if (hts_set_fai_filename(state->fp, opts->ga.reference) != 0) {
- print_error_errno("bam2fq", "cannot load reference \"%s\"", opts->ga.reference);
- free(state);
- return false;
- }
- }
-
- int i;
- for (i = 0; i < 3; ++i) {
- if (opts->fnr[i]) {
- state->fpr[i] = open_fqfile(opts->fnr[i], state->compression_level);
- if (state->fpr[i] == NULL) {
- print_error_errno("bam2fq", "Cannot write to r%d file \"%s\"", i, opts->fnr[i]);
- free(state);
- return false;
- }
- } else {
- if (!state->hsamtools_stdout) {
- state->hsamtools_stdout = bgzf_dopen(fileno(samtools_stdout), "wu");
- if (!state->hsamtools_stdout) {
- print_error_errno("bam2fq", "Cannot open STDOUT");
- free(state);
- return false;
- }
- }
- state->fpr[i] = state->hsamtools_stdout;
- }
- }
- for (i = 0; i < 2; i++) {
- state->fpi[i] = NULL;
- if (opts->index_file[i]) {
- state->fpi[i] = open_fqfile(opts->index_file[i], state->compression_level);
- if (state->fpi[i] == NULL) {
- print_error_errno("bam2fq", "Cannot write to i%d file \"%s\"", i+1, opts->index_file[i]);
- free(state);
- return false;
- }
- }
- }
-
- state->h = sam_hdr_read(state->fp);
- if (state->h == NULL) {
- fprintf(samtools_stderr, "Failed to read header for \"%s\"\n", opts->fn_input);
- free(state);
- return false;
- }
-
- *state_out = state;
- return true;
-}
-
-static bool destroy_state(const bam2fq_opts_t *opts, bam2fq_state_t *state, int* status)
-{
- bool valid = true;
- bam_hdr_destroy(state->h);
- check_sam_close("bam2fq", state->fp, opts->fn_input, "file", status);
- if (state->fpse && bgzf_close(state->fpse)) { print_error_errno("bam2fq", "Error closing singleton file \"%s\"", opts->fnse); valid = false; }
- int i;
- for (i = 0; i < 3; ++i) {
- if (state->fpr[i] != state->hsamtools_stdout) {
- if (bgzf_close(state->fpr[i])) { print_error_errno("bam2fq", "Error closing r%d file \"%s\"", i, opts->fnr[i]); valid = false; }
- }
- }
- if (state->hsamtools_stdout) {
- if (bgzf_close(state->hsamtools_stdout)) {
- print_error_errno("bam2fq", "Error closing STDOUT");
- valid = false;
- }
- }
- for (i = 0; i < 2; i++) {
- if (state->fpi[i] && bgzf_close(state->fpi[i])) {
- print_error_errno("bam2fq", "Error closing i%d file \"%s\"", i+1, opts->index_file[i]);
- valid = false;
- }
- }
- kl_destroy(ktaglist,state->taglist);
- free(state->index_sequence);
- free(state);
- return valid;
-}
-
-static inline bool filter_it_out(const bam1_t *b, const bam2fq_state_t *state)
-{
- return (b->core.flag&(BAM_FSECONDARY|BAM_FSUPPLEMENTARY) // skip secondary and supplementary alignments
- || (b->core.flag&(state->flag_on)) != state->flag_on // or reads indicated by filter flags
- || (b->core.flag&(state->flag_off)) != 0
- || (b->core.flag&(state->flag_alloff) && (b->core.flag&(state->flag_alloff)) == state->flag_alloff));
-
-}
-
-static bool bam2fq_mainloop(bam2fq_state_t *state, bam2fq_opts_t* opts)
-{
- int n;
- bam1_t *records[3];
- bam1_t* b = bam_init1();
- char *current_qname = NULL;
- int64_t n_reads = 0, n_singletons = 0; // Statistics
- kstring_t linebuf[3] = {{0,0,NULL},{0,0,NULL},{0,0,NULL}};
- int score[3];
- int at_eof;
- if (b == NULL ) {
- perror("[bam2fq_mainloop] Malloc error for bam record buffer.");
- return false;
- }
-
- bool valid = true;
- while (true) {
- int res = sam_read1(state->fp, state->h, b);
- if (res < -1) {
- fprintf(samtools_stderr, "[bam2fq_mainloop] Failed to read bam record.\n");
- return false;
- }
- at_eof = res < 0;
-
- if (!at_eof && filter_it_out(b, state)) continue;
- if (!at_eof) ++n_reads;
-
- if (at_eof || !current_qname || (strcmp(current_qname, bam_get_qname(b)) != 0)) {
- if (current_qname) {
- if (state->illumina_tag) {
- for (n=0; valid && n<3; n++) {
- if (insert_index_sequence_into_linebuf(state->index_sequence, &linebuf[n], records[n]) < 0) valid = false;
- }
- if (!valid) break;
- }
- free(state->index_sequence); state->index_sequence = NULL;
- if (score[1] > 0 && score[2] > 0) {
- // print linebuf[1] to fpr[1], linebuf[2] to fpr[2]
- if (bgzf_write(state->fpr[1], linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
- if (bgzf_write(state->fpr[2], linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
- } else if (score[1] > 0 || score[2] > 0) {
- if (state->fpse) {
- // print whichever one exists to fpse
- if (score[1] > 0) {
- if (bgzf_write(state->fpse, linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
- } else {
- if (bgzf_write(state->fpse, linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
- }
- ++n_singletons;
- } else {
- if (score[1] > 0) {
- if (bgzf_write(state->fpr[1], linebuf[1].s, linebuf[1].l) < 0) { valid = false; break; }
- } else {
- if (bgzf_write(state->fpr[2], linebuf[2].s, linebuf[2].l) < 0) { valid = false; break; }
- }
- }
- }
- if (score[0]) { // TODO: check this
- // print linebuf[0] to fpr[0]
- if (bgzf_write(state->fpr[0], linebuf[0].s, linebuf[0].l) < 0) { valid = false; break; }
- }
- }
-
- if (at_eof) break;
-
- free(current_qname);
- current_qname = strdup(bam_get_qname(b));
- if (!current_qname) { valid = false; break; }
- score[0] = score[1] = score[2] = 0;
- }
-
- // Prefer a copy of the read that has base qualities
- int b_score = bam_get_qual(b)[0] != 0xff? 2 : 1;
- if (b_score > score[which_readpart(b)]) {
- if (state->fpi[0]) if (!tags2fq(b, state, opts)) return false;
- records[which_readpart(b)] = b;
- if(!bam1_to_fq(b, &linebuf[which_readpart(b)], state)) {
- fprintf(samtools_stderr, "[%s] Error converting read to FASTA/Q\n", __func__);
- return false;
- }
- score[which_readpart(b)] = b_score;
- }
- }
- if (!valid)
- {
- perror("[bam2fq_mainloop] Error writing to FASTx files.");
- }
- bam_destroy1(b);
- free(current_qname);
- free(linebuf[0].s);
- free(linebuf[1].s);
- free(linebuf[2].s);
- fprintf(samtools_stderr, "[M::%s] discarded %" PRId64 " singletons\n", __func__, n_singletons);
- fprintf(samtools_stderr, "[M::%s] processed %" PRId64 " reads\n", __func__, n_reads);
-
- return valid;
-}
-
-int main_bam2fq(int argc, char *argv[])
-{
- int status = EXIT_SUCCESS;
- bam2fq_opts_t* opts = NULL;
- bam2fq_state_t* state = NULL;
-
- bool valid = parse_opts(argc, argv, &opts);
- if (!valid || opts == NULL) return valid ? EXIT_SUCCESS : EXIT_FAILURE;
-
- if (!init_state(opts, &state)) return EXIT_FAILURE;
-
- if (!bam2fq_mainloop(state,opts)) status = EXIT_FAILURE;
-
- if (!destroy_state(opts, state, &status)) return EXIT_FAILURE;
- sam_global_args_free(&opts->ga);
- free_opts(opts);
-
- return status;
-}
/* samtools.h -- utility routines.
- Copyright (C) 2013-2015 Genome Research Ltd.
+ Copyright (C) 2013-2015, 2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
#ifndef SAMTOOLS_H
#define SAMTOOLS_H
+#include "htslib/hts_defs.h"
+#include "htslib/sam.h"
+
const char *samtools_version(void);
-#if defined __GNUC__ && __GNUC__ >= 2
-#define CHECK_PRINTF(fmt,args) __attribute__ ((format (printf, fmt, args)))
-#else
-#define CHECK_PRINTF(fmt,args)
-#endif
+#define CHECK_PRINTF(fmt,args) HTS_FORMAT(HTS_PRINTF_FMT, (fmt), (args))
void print_error(const char *subcommand, const char *format, ...) CHECK_PRINTF(2, 3);
void print_error_errno(const char *subcommand, const char *format, ...) CHECK_PRINTF(2, 3);
+void check_sam_close(const char *subcmd, samFile *fp, const char *fname, const char *null_fname, int *retp);
+
+/*
+ * Utility function to add an index to a file we've opened for write.
+ * NB: Call this after writing the header and before writing sequences.
+ *
+ * The returned index filename should be freed by the caller, but only
+ * after sam_idx_save has been called.
+ *
+ * Returns index filename on success,
+ * NULL on failure.
+ */
+char *auto_index(htsFile *fp, const char *fn, bam_hdr_t *header);
+
#endif
/* stats.c -- This is the former bamcheck integrated into samtools/htslib.
- Copyright (C) 2012-2015 Genome Research Ltd.
+ Copyright (C) 2012-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
Author: Sam Nicholls <sam@samnicholls.net>
#include <string.h>
#include <math.h>
#include <ctype.h>
+#include <inttypes.h>
#include <getopt.h>
#include <errno.h>
#include <assert.h>
#include <htslib/faidx.h>
#include <htslib/sam.h>
#include <htslib/hts.h>
-#include "sam_header.h"
+#include <htslib/hts_defs.h>
#include <htslib/khash_str2int.h>
#include "samtools.h"
#include <htslib/khash.h>
#define BWA_MIN_RDLEN 35
#define DEFAULT_CHUNK_NO 8
#define DEFAULT_PAIR_MAX 10000
+#define ERROR_LIMIT 200
// From the spec
// If 0x4 is set, no assumptions can be made about RNAME, POS, CIGAR, MAPQ, bits 0x2, 0x10, 0x100 and 0x800, and the bit 0x20 of the previous read in the template.
+#define IS_PAIRED(bam) ((bam)->core.flag&BAM_FPAIRED)
#define IS_PAIRED_AND_MAPPED(bam) (((bam)->core.flag&BAM_FPAIRED) && !((bam)->core.flag&BAM_FUNMAP) && !((bam)->core.flag&BAM_FMUNMAP))
#define IS_PROPERLYPAIRED(bam) (((bam)->core.flag&(BAM_FPAIRED|BAM_FPROPER_PAIR)) == (BAM_FPAIRED|BAM_FPROPER_PAIR) && !((bam)->core.flag&BAM_FUNMAP))
#define IS_UNMAPPED(bam) ((bam)->core.flag&BAM_FUNMAP)
#define IS_DUP(bam) ((bam)->core.flag&BAM_FDUP)
#define IS_ORIGINAL(bam) (((bam)->core.flag&(BAM_FSECONDARY|BAM_FSUPPLEMENTARY)) == 0)
+#define READ_ORDER_NONE 0
+#define READ_ORDER_FIRST 1
+#define READ_ORDER_LAST 2
+#define READ_ORDER_MIDDLE 3
+
+#define REG_INC 100
+#define POS_INC 1000
+
// The GC-depth graph works as follows: split the reference sequence into
// segments and calculate GC content and depth in each bin. Then sort
// these segments by their GC and plot the depth distribution by means
// For coverage distribution, a simple pileup
typedef struct
{
- int64_t pos;
+ hts_pos_t pos;
int size, start;
int *buffer;
}
round_buffer_t;
-typedef struct { uint32_t from, to; } pos_t;
typedef struct
{
- int npos,mpos,cpos;
- pos_t *pos;
+ int npos, mpos, cpos;
+ hts_pair_pos_t *pos;
}
regions_t;
}
acgtno_count_t;
+typedef struct
+{
+ char tag_name[3];
+ char qual_name[3];
+ uint32_t nbases;
+ int32_t tag_sep; // Index of the separator (if present)
+ int32_t max_qual;
+ uint32_t offset; // Where the tag stats info is located in the allocated memory
+}
+barcode_info_t;
+
typedef struct
{
// Auxiliary data
float isize_main_bulk; // There are always some unrealistically big insert sizes, report only the main part
int cov_min,cov_max,cov_step; // Minimum, maximum coverage and size of the coverage bins
samFile* sam;
- bam_hdr_t* sam_header;
+ sam_hdr_t* sam_header;
// Filters
int filter_readlen;
uint64_t total_len_dup;
uint64_t nreads_1st;
uint64_t nreads_2nd;
+ uint64_t nreads_other;
uint64_t nreads_filtered;
uint64_t nreads_dup;
uint64_t nreads_unmapped;
// GC-depth related data
uint32_t ngcd, igcd; // The maximum number of GC depth bins and index of the current bin
gc_depth_t *gcd; // The GC-depth bins holder
- int32_t tid, gcd_pos; // Position of the current bin
- int32_t pos; // Position of the last read
+ int32_t tid; // Position of the current bin
+ hts_pos_t gcd_pos, pos; // Position of the last read
// Coverage distribution related data
int ncov; // The number of coverage bins
// Mismatches by read cycle
uint8_t *rseq_buf; // A buffer for reference sequence to check the mismatches against
int mrseq_buf; // The size of the buffer
- int32_t rseq_pos; // The coordinate of the first base in the buffer
- int32_t nrseq_buf; // The used part of the buffer
+ hts_pos_t rseq_pos; // The coordinate of the first base in the buffer
+ int64_t nrseq_buf; // The used part of the buffer
uint64_t *mpc_buf; // Mismatches per cycle
// Target regions
- int nregions, reg_from, reg_to;
+ int nregions;
+ hts_pos_t reg_from, reg_to;
regions_t *regions;
// Auxiliary data
char* split_name;
stats_info_t* info; // Pointer to options and settings struct
- pos_t *chunks;
+ hts_pair_pos_t *chunks;
uint32_t nchunks;
uint32_t pair_count; // Number of active pairs in the pairing hash table
uint32_t target_count; // Number of bases covered by the target file
uint32_t last_pair_tid;
uint32_t last_read_flush;
+
+ // Barcode statistics
+ acgtno_count_t *acgtno_barcode;
+ uint64_t *quals_barcode;
+ barcode_info_t *tags_barcode;
+ uint32_t ntags;
+ uint32_t error_number;
}
stats_t;
KHASH_MAP_INIT_STR(c2stats, stats_t*)
typedef struct {
uint32_t first; // 1 - first read, 2 - second read
uint32_t n, m; // number of chunks, allocated chunks
- pos_t *chunks; // chunk array of size m
+ hts_pair_pos_t *chunks; // chunk array of size m
} pair_t;
KHASH_MAP_INIT_STR(qn2pair, pair_t*)
-static void error(const char *format, ...);
+static void HTS_NORETURN error(const char *format, ...);
int is_in_regions(bam1_t *bam_line, stats_t *stats);
void realloc_buffers(stats_t *stats, int seq_len);
static int regions_lt(const void *r1, const void *r2) {
- int64_t from_diff = (int64_t)((pos_t *)r1)->from - (int64_t)((pos_t *)r2)->from;
- int64_t to_diff = (int64_t)((pos_t *)r1)->to - (int64_t)((pos_t *)r2)->to;
+ int64_t from_diff = ((hts_pair_pos_t *)r1)->beg - ((hts_pair_pos_t *)r2)->beg;
+ int64_t to_diff = ((hts_pair_pos_t *)r1)->end - ((hts_pair_pos_t *)r2)->end;
return from_diff > 0 ? 1 : from_diff < 0 ? -1 : to_diff > 0 ? 1 : to_diff < 0 ? -1 : 0;
}
return 1 + (depth - min) / step;
}
-static inline int round_buffer_lidx2ridx(int offset, int size, int64_t refpos, int64_t pos)
+static inline int round_buffer_lidx2ridx(int offset, int size, hts_pos_t refpos, hts_pos_t pos)
{
return (offset + (pos-refpos) % size) % size;
}
-void round_buffer_flush(stats_t *stats, int64_t pos)
+void round_buffer_flush(stats_t *stats, hts_pos_t pos)
{
int ibuf,idp;
if ( pos==stats->cov_rbuf.pos )
return;
- int64_t new_pos = pos;
+ hts_pos_t new_pos = pos;
if ( pos==-1 || pos - stats->cov_rbuf.pos >= stats->cov_rbuf.size )
{
// Flush the whole buffer, but in sequential order,
}
if ( pos < stats->cov_rbuf.pos )
- error("Expected coordinates in ascending order, got %ld after %ld\n", pos,stats->cov_rbuf.pos);
+ error("Expected coordinates in ascending order, got %"PRIhts_pos" after %"PRIhts_pos"\n", pos, stats->cov_rbuf.pos);
int ifrom = stats->cov_rbuf.start;
- int ito = round_buffer_lidx2ridx(stats->cov_rbuf.start,stats->cov_rbuf.size,stats->cov_rbuf.pos,pos-1);
+ int ito = round_buffer_lidx2ridx(stats->cov_rbuf.start, stats->cov_rbuf.size, stats->cov_rbuf.pos, pos-1);
if ( ifrom>ito )
{
for (ibuf=ifrom; ibuf<stats->cov_rbuf.size; ibuf++)
stats->cov[idp]++;
stats->cov_rbuf.buffer[ibuf] = 0;
}
- stats->cov_rbuf.start = (new_pos==-1) ? 0 : round_buffer_lidx2ridx(stats->cov_rbuf.start,stats->cov_rbuf.size,stats->cov_rbuf.pos,pos);
+ stats->cov_rbuf.start = (new_pos==-1) ? 0 : round_buffer_lidx2ridx(stats->cov_rbuf.start, stats->cov_rbuf.size, stats->cov_rbuf.pos, pos);
stats->cov_rbuf.pos = new_pos;
}
-void round_buffer_insert_read(round_buffer_t *rbuf, int64_t from, int64_t to)
+/**
+ * [from, to) - 0 based half-open
+ */
+static void round_buffer_insert_read(round_buffer_t *rbuf, hts_pos_t from, hts_pos_t to)
{
- if ( to-from >= rbuf->size )
- error("The read length too big (%d), please increase the buffer length (currently %d)\n", to-from+1,rbuf->size);
+ if ( to-from > rbuf->size )
+ error("The read length too big (%"PRIhts_pos"), please increase the buffer length (currently %d)\n", to-from, rbuf->size);
if ( from < rbuf->pos )
- error("The reads are not sorted (%ld comes after %ld).\n", from,rbuf->pos);
+ error("The reads are not sorted (%"PRIhts_pos" comes after %"PRIhts_pos").\n", from, rbuf->pos);
- int ifrom,ito,ibuf;
- ifrom = round_buffer_lidx2ridx(rbuf->start,rbuf->size,rbuf->pos,from);
- ito = round_buffer_lidx2ridx(rbuf->start,rbuf->size,rbuf->pos,to);
+ int ifrom, ito, ibuf;
+ ifrom = round_buffer_lidx2ridx(rbuf->start, rbuf->size, rbuf->pos, from);
+ ito = round_buffer_lidx2ridx(rbuf->start, rbuf->size, rbuf->pos, to);
if ( ifrom>ito )
{
for (ibuf=ifrom; ibuf<rbuf->size; ibuf++)
rbuf->buffer[ibuf]++;
ifrom = 0;
}
- for (ibuf=ifrom; ibuf<=ito; ibuf++)
+ for (ibuf=ifrom; ibuf<ito; ibuf++)
rbuf->buffer[ibuf]++;
}
void count_indels(stats_t *stats,bam1_t *bam_line)
{
int is_fwd = IS_REVERSE(bam_line) ? 0 : 1;
- int is_1st = IS_READ1(bam_line) ? 1 : 0;
+ uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
int icig;
int icycle = 0;
int read_len = bam_line->core.l_qseq;
int idx = is_fwd ? icycle : read_len-icycle-ncig;
if ( idx<0 )
error("FIXME: read_len=%d vs icycle=%d\n", read_len,icycle);
- if ( idx >= stats->nbases || idx<0 ) error("FIXME: %d vs %d, %s:%d %s\n", idx,stats->nbases, stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam_get_qname(bam_line));
- if ( is_1st )
+ if ( idx >= stats->nbases || idx<0 ) error("FIXME: %d vs %d, %s:%"PRIhts_pos" %s\n", idx, stats->nbases, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
+ if ( order == READ_ORDER_FIRST )
stats->ins_cycles_1st[idx]++;
- else
+ if ( order == READ_ORDER_LAST )
stats->ins_cycles_2nd[idx]++;
icycle += ncig;
if ( ncig<=stats->nindels )
int idx = is_fwd ? icycle-1 : read_len-icycle-1;
if ( idx<0 ) continue; // discard meaningless deletions
if ( idx >= stats->nbases ) error("FIXME: %d vs %d\n", idx,stats->nbases);
- if ( is_1st )
+ if ( order == READ_ORDER_FIRST )
stats->del_cycles_1st[idx]++;
- else
+ if ( order == READ_ORDER_LAST )
stats->del_cycles_2nd[idx]++;
if ( ncig<=stats->nindels )
stats->deletions[ncig-1]++;
void count_mismatches_per_cycle(stats_t *stats, bam1_t *bam_line, int read_len)
{
int is_fwd = IS_REVERSE(bam_line) ? 0 : 1;
- int icig,iread=0,icycle=0;
- int iref = bam_line->core.pos - stats->rseq_pos;
+ int icig, iread=0, icycle=0;
+ hts_pos_t iref = bam_line->core.pos - stats->rseq_pos;
uint8_t *read = bam_get_seq(bam_line);
uint8_t *quals = bam_get_qual(bam_line);
uint64_t *mpc_buf = stats->mpc_buf;
continue;
}
// Ignore H and N CIGARs. The letter are inserted e.g. by TopHat and often require very large
- // chunk of refseq in memory. Not very frequent and not noticable in the stats.
+ // chunk of refseq in memory. Not very frequent and not noticeable in the stats.
if ( cig==BAM_CREF_SKIP || cig==BAM_CHARD_CLIP || cig==BAM_CPAD ) continue;
if ( cig!=BAM_CMATCH && cig!=BAM_CEQUAL && cig!=BAM_CDIFF ) // not relying on precalculated diffs
- error("TODO: cigar %d, %s:%d %s\n", cig,stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam_get_qname(bam_line));
+ error("TODO: cigar %d, %s:%"PRIhts_pos" %s\n", cig, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
if ( ncig+iref > stats->nrseq_buf )
- error("FIXME: %d+%d > %d, %s, %s:%d\n",ncig,iref,stats->nrseq_buf, bam_get_qname(bam_line),stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1);
+ error("FIXME: %d+%"PRIhts_pos" > %"PRId64", %s, %s:%"PRIhts_pos"\n", ncig, iref, stats->nrseq_buf, bam_get_qname(bam_line), sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1);
int im;
for (im=0; im<ncig; im++)
{
uint8_t qual = quals[iread] + 1;
if ( qual>=stats->nquals )
- error("TODO: quality too high %d>=%d (%s %d %s)\n", qual,stats->nquals, stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam_get_qname(bam_line));
+ error("TODO: quality too high %d>=%d (%s %"PRIhts_pos" %s)\n", qual, stats->nquals, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
int idx = is_fwd ? icycle : read_len-icycle-1;
if ( idx>stats->max_len )
- error("mpc: %d>%d (%s %d %s)\n",idx,stats->max_len,stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam_get_qname(bam_line));
+ error("mpc: %d>%d (%s %"PRIhts_pos" %s)\n", idx, stats->max_len, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
idx = idx*stats->nquals + qual;
if ( idx>=stats->nquals*stats->nbases )
}
}
-void read_ref_seq(stats_t *stats, int32_t tid, int32_t pos)
+void read_ref_seq(stats_t *stats, int32_t tid, hts_pos_t pos)
{
- int i, fai_ref_len;
- char *fai_ref = faidx_fetch_seq(stats->info->fai, stats->info->sam_header->target_name[tid], pos, pos+stats->mrseq_buf-1, &fai_ref_len);
- if ( fai_ref_len<0 ) error("Failed to fetch the sequence \"%s\"\n", stats->info->sam_header->target_name[tid]);
+ int i;
+ hts_pos_t fai_ref_len;
+ char *fai_ref = faidx_fetch_seq64(stats->info->fai, sam_hdr_tid2name(stats->info->sam_header, tid), pos, pos+stats->mrseq_buf-1, &fai_ref_len);
+ if ( fai_ref_len < 0 ) error("Failed to fetch the sequence \"%s\"\n", sam_hdr_tid2name(stats->info->sam_header, tid));
uint8_t *ptr = stats->rseq_buf;
for (i=0; i<fai_ref_len; i++)
stats->tid = tid;
}
-float fai_gc_content(stats_t *stats, int pos, int len)
+float fai_gc_content(stats_t *stats, hts_pos_t pos, int len)
{
uint32_t gc,count,c;
- int i = pos - stats->rseq_pos, ito = i + len;
+ hts_pos_t i = pos - stats->rseq_pos, ito = i + len;
assert( i>=0 );
if ( ito > stats->nrseq_buf ) ito = stats->nrseq_buf;
if ( stats->mrseq_buf<n )
{
stats->rseq_buf = realloc(stats->rseq_buf,sizeof(uint8_t)*n);
+ if (!stats->rseq_buf) {
+ error("Could not reallocate reference sequence buffer");
+ }
stats->mrseq_buf = n;
}
}
// Realloc the coverage distribution buffer
int *rbuffer = calloc(sizeof(int),seq_len*5);
+ if (!rbuffer) {
+ error("Could not allocate coverage distribution buffer");
+ }
n = stats->cov_rbuf.size-stats->cov_rbuf.start;
memcpy(rbuffer,stats->cov_rbuf.buffer+stats->cov_rbuf.start,n);
if ( stats->cov_rbuf.start>1 )
stats->checksum.quals += crc32(0L, qual, (seq_len+1)/2);
}
+// Collect statistics about the barcode tags specified by init_barcode_tags method
+static void collect_barcode_stats(bam1_t* bam_line, stats_t* stats) {
+ uint32_t nbases, tag, i;
+ acgtno_count_t *acgtno;
+ uint64_t *quals;
+ int32_t *separator, *maxqual;
+
+ for (tag = 0; tag < stats->ntags; tag++) {
+ const char *barcode_tag = stats->tags_barcode[tag].tag_name, *qual_tag = stats->tags_barcode[tag].qual_name;
+ uint8_t* bc = bam_aux_get(bam_line, barcode_tag);
+ if (!bc)
+ continue;
+
+ char* barcode = bam_aux2Z(bc);
+ if (!barcode)
+ continue;
+
+ uint32_t barcode_len = strlen(barcode);
+ if (!stats->tags_barcode[tag].nbases) { // tag seen for the first time
+ uint32_t offset = 0;
+ for (i = 0; i < stats->ntags; i++)
+ offset += stats->tags_barcode[i].nbases;
+
+ stats->tags_barcode[tag].offset = offset;
+ stats->tags_barcode[tag].nbases = barcode_len;
+ stats->acgtno_barcode = realloc(stats->acgtno_barcode, (offset + barcode_len) * sizeof(acgtno_count_t));
+ stats->quals_barcode = realloc(stats->quals_barcode, (offset + barcode_len) * stats->nquals * sizeof(uint64_t));
+
+ if (!stats->acgtno_barcode || !stats->quals_barcode)
+ error("Error allocating memory. Aborting!\n");
+
+ memset(stats->acgtno_barcode + offset, 0, barcode_len*sizeof(acgtno_count_t));
+ memset(stats->quals_barcode + offset*stats->nquals, 0, barcode_len*stats->nquals*sizeof(uint64_t));
+ }
+
+ nbases = stats->tags_barcode[tag].nbases;
+ if (barcode_len > nbases) {
+ fprintf(stderr, "Barcodes with tag %s differ in length at sequence '%s'\n", barcode_tag, bam_get_qname(bam_line));
+ continue;
+ }
+
+ acgtno = stats->acgtno_barcode + stats->tags_barcode[tag].offset;
+ quals = stats->quals_barcode + stats->tags_barcode[tag].offset*stats->nquals;
+ maxqual = &stats->tags_barcode[tag].max_qual;
+ separator = &stats->tags_barcode[tag].tag_sep;
+ int error_flag = 0;
+
+ for (i = 0; i < barcode_len; i++) {
+ switch (barcode[i]) {
+ case 'A':
+ acgtno[i].a++;
+ break;
+ case 'C':
+ acgtno[i].c++;
+ break;
+ case 'G':
+ acgtno[i].g++;
+ break;
+ case 'T':
+ acgtno[i].t++;
+ break;
+ case 'N':
+ acgtno[i].n++;
+ break;
+ default:
+ if (*separator >= 0) {
+ if (*separator != i) {
+ if (stats->error_number < ERROR_LIMIT) {
+ fprintf(stderr, "Barcode separator for tag %s is in a different position or wrong barcode content('%s') at sequence '%s'\n", barcode_tag, barcode, bam_get_qname(bam_line));
+ stats->error_number++;
+ }
+ error_flag = 1;
+ }
+ } else {
+ *separator = i;
+ }
+ }
+
+ /* don't process the rest of the tag bases */
+ if (error_flag)
+ break;
+ }
+
+ /* skip to the next tag */
+ if (error_flag)
+ continue;
+
+ uint8_t* qt = bam_aux_get(bam_line, qual_tag);
+ if (!qt)
+ continue;
+
+ char* barqual = bam_aux2Z(qt);
+ if (!barqual)
+ continue;
+
+ uint32_t barqual_len = strlen(barqual);
+ if (barqual_len == barcode_len) {
+ for (i = 0; i < barcode_len; i++) {
+ int32_t qual = (int32_t)barqual[i] - '!'; // Phred + 33
+ if (qual >= 0 && qual < stats->nquals) {
+ quals[i * stats->nquals + qual]++;
+ if (qual > *maxqual)
+ *maxqual = qual;
+ }
+ }
+ } else {
+ if (stats->error_number++ < ERROR_LIMIT) {
+ fprintf(stderr, "%s length and %s length don't match for sequence '%s'\n", barcode_tag, qual_tag, bam_get_qname(bam_line));
+ }
+ }
+ }
+}
+
// These stats should only be calculated for the original reads ignoring
// supplementary artificial reads otherwise we'll accidentally double count
void collect_orig_read_stats(bam1_t *bam_line, stats_t *stats, int* gc_count_out)
if ( bam_line->core.flag & BAM_FQCFAIL ) stats->nreads_QCfailed++;
if ( bam_line->core.flag & BAM_FPAIRED ) stats->nreads_paired_tech++;
+ uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
+
// Count GC and ACGT per cycle. Note that cycle is approximate, clipping is ignored
uint8_t *seq = bam_get_seq(bam_line);
- int i, read_cycle, gc_count = 0, reverse = IS_REVERSE(bam_line), is_first = IS_READ1(bam_line);
- for (i=0; i<seq_len; i++)
- {
- // Read cycle for current index
- read_cycle = (reverse ? seq_len-i-1 : i);
+ int i, read_cycle, gc_count = 0, reverse = IS_REVERSE(bam_line);
- // Conversion from uint8_t coding:
- // -12-4---8------5
- // =ACMGRSVTWYHKDBN
- switch (bam_seqi(seq, i)) {
- case 1:
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].a++ : stats->acgtno_cycles_2nd[ read_cycle ].a++;
- break;
- case 2:
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].c++ : stats->acgtno_cycles_2nd[ read_cycle ].c++;
- gc_count++;
- break;
- case 4:
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].g++ : stats->acgtno_cycles_2nd[ read_cycle ].g++;
- gc_count++;
- break;
- case 8:
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].t++ : stats->acgtno_cycles_2nd[ read_cycle ].t++;
- break;
- case 15:
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].n++ : stats->acgtno_cycles_2nd[ read_cycle ].n++;
- break;
- default:
- /*
- * count "=" sequences in "other" along
- * with MRSVWYHKDB ambiguity codes
- */
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].other++ : stats->acgtno_cycles_2nd[ read_cycle ].other++;
- break;
+ acgtno_count_t *acgtno_cycles = (order == READ_ORDER_FIRST) ? stats->acgtno_cycles_1st : (order == READ_ORDER_LAST) ? stats->acgtno_cycles_2nd : NULL ;
+ if (acgtno_cycles) {
+ for (i=0; i<seq_len; i++)
+ {
+ // Read cycle for current index
+ read_cycle = (reverse ? seq_len-i-1 : i);
+
+ // Conversion from uint8_t coding:
+ // -12-4---8------5
+ // =ACMGRSVTWYHKDBN
+ switch (bam_seqi(seq, i)) {
+ case 1:
+ acgtno_cycles[ read_cycle ].a++;
+ break;
+ case 2:
+ acgtno_cycles[ read_cycle ].c++;
+ gc_count++;
+ break;
+ case 4:
+ acgtno_cycles[ read_cycle ].g++;
+ gc_count++;
+ break;
+ case 8:
+ acgtno_cycles[ read_cycle ].t++;
+ break;
+ case 15:
+ acgtno_cycles[ read_cycle ].n++;
+ break;
+ default:
+ /*
+ * count "=" sequences in "other" along
+ * with MRSVWYHKDB ambiguity codes
+ */
+ acgtno_cycles[ read_cycle ].other++;
+ break;
+ }
}
}
int gc_idx_min = gc_count*(stats->ngc-1)/seq_len;
// Determine which array (1st or 2nd read) will these stats go to,
// trim low quality bases from end the same way BWA does,
// fill GC histogram
- uint64_t *quals;
+ uint64_t *quals = NULL;
uint8_t *bam_quals = bam_get_qual(bam_line);
- if ( IS_READ2(bam_line) )
- {
- quals = stats->quals_2nd;
- stats->nreads_2nd++;
- stats->total_len_2nd += seq_len;
- for (i=gc_idx_min; i<gc_idx_max; i++)
- stats->gc_2nd[i]++;
- }
- else
- {
+
+ switch (order) {
+ case READ_ORDER_FIRST:
quals = stats->quals_1st;
stats->nreads_1st++;
stats->total_len_1st += seq_len;
for (i=gc_idx_min; i<gc_idx_max; i++)
stats->gc_1st[i]++;
+ break;
+ case READ_ORDER_LAST:
+ quals = stats->quals_2nd;
+ stats->nreads_2nd++;
+ stats->total_len_2nd += seq_len;
+ for (i=gc_idx_min; i<gc_idx_max; i++)
+ stats->gc_2nd[i]++;
+ break;
+ default:
+ stats->nreads_other++;
}
if ( stats->info->trim_qual>0 )
stats->nbases_trimmed += bwa_trim_read(stats->info->trim_qual, bam_quals, seq_len, reverse);
// Quality histogram and average quality. Clipping is neglected.
- for (i=0; i<seq_len; i++)
- {
- uint8_t qual = bam_quals[ reverse ? seq_len-i-1 : i];
- if ( qual>=stats->nquals )
- error("TODO: quality too high %d>=%d (%s %d %s)\n", qual,stats->nquals,stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam_get_qname(bam_line));
- if ( qual>stats->max_qual )
- stats->max_qual = qual;
+ if (quals) {
+ for (i=0; i<seq_len; i++)
+ {
+ uint8_t qual = bam_quals[ reverse ? seq_len-i-1 : i];
+ if ( qual>=stats->nquals )
+ error("TODO: quality too high %d>=%d (%s %"PRIhts_pos" %s)\n", qual, stats->nquals, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
+ if ( qual>stats->max_qual )
+ stats->max_qual = qual;
+
+ quals[ i*stats->nquals+qual ]++;
+ stats->sum_qual += qual;
+ }
+ }
- quals[ i*stats->nquals+qual ]++;
- stats->sum_qual += qual;
+ // Barcode statistics
+ if (order == READ_ORDER_FIRST) {
+ collect_barcode_stats(bam_line, stats);
}
// Look at the flags and increment appropriate counters (mapped, paired, etc)
*gc_count_out = gc_count;
}
-static int cleanup_overlaps(khash_t(qn2pair) *read_pairs, int max) {
+static int cleanup_overlaps(khash_t(qn2pair) *read_pairs, hts_pos_t max) {
if ( !read_pairs )
return 0;
char *key = (char *)kh_key(read_pairs, k);
pair_t *val = kh_val(read_pairs, k);
if ( val && val->chunks ) {
- if ( val->chunks[val->n-1].to < max ) {
+ if ( val->chunks[val->n-1].end < max ) {
free(val->chunks);
free(val);
free(key);
}
}
}
- if ( max == INT_MAX )
+ if ( max == INT64_MAX )
kh_destroy(qn2pair, read_pairs);
return count;
}
-static void remove_overlaps(bam1_t *bam_line, khash_t(qn2pair) *read_pairs, stats_t *stats, int pmin, int pmax) {
+/**
+ * [pmin, pmax) - 0 based half-open
+ */
+static void remove_overlaps(bam1_t *bam_line, khash_t(qn2pair) *read_pairs, stats_t *stats, hts_pos_t pmin, hts_pos_t pmax) {
if ( !bam_line || !read_pairs || !stats )
return;
- uint32_t first = (IS_READ1(bam_line) > 0 ? 1 : 0) + (IS_READ2(bam_line) > 0 ? 2 : 0) ;
+ uint32_t order = (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0);
if ( !(bam_line->core.flag & BAM_FPAIRED) ||
(bam_line->core.flag & BAM_FMUNMAP) ||
- (abs(bam_line->core.isize) >= 2*bam_line->core.l_qseq) ||
- (first != 1 && first != 2) ) {
+ (llabs(bam_line->core.isize) >= 2*bam_line->core.l_qseq) ||
+ (order != READ_ORDER_FIRST && order != READ_ORDER_LAST) ) {
if ( pmin >= 0 )
- round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax-1);
+ round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
return;
}
char *qname = bam_get_qname(bam_line);
if ( !qname ) {
- fprintf(stderr, "Error retrieving qname for line starting at pos %d\n", bam_line->core.pos);
+ fprintf(stderr, "Error retrieving qname for line starting at pos %"PRIhts_pos"\n", bam_line->core.pos);
return;
}
k = kh_put(qn2pair, read_pairs, s, &ret);
if ( -1 == ret ) {
- fprintf(stderr, "Error inserting read '%s' in pair hash table\n", qname);
- return;
+ error("Error inserting read '%s' in pair hash table\n", qname);
}
pair_t *pc = calloc(1, sizeof(pair_t));
}
pc->m = DEFAULT_CHUNK_NO;
- pc->chunks = calloc(pc->m, sizeof(pos_t));
+ pc->chunks = calloc(pc->m, sizeof(hts_pair_pos_t));
if ( !pc->chunks ) {
fprintf(stderr, "Error allocating memory\n");
return;
}
- pc->chunks[0].from = pmin;
- pc->chunks[0].to = pmax;
+ pc->chunks[0].beg = pmin;
+ pc->chunks[0].end = pmax;
pc->n = 1;
- pc->first = first;
+ pc->first = order;
kh_val(read_pairs, k) = pc;
stats->pair_count++;
return;
}
- if ( first == pc->first ) { //chunk from an existing line
+ if ( order == pc->first ) { //chunk from an existing line
if ( pmin == -1 )
return;
if ( pc->n == pc->m ) {
- pos_t *tmp = realloc(pc->chunks, (pc->m<<1)*sizeof(pos_t));
+ hts_pair_pos_t *tmp = realloc(pc->chunks, (pc->m<<1)*sizeof(hts_pair_pos_t));
if ( !tmp ) {
fprintf(stderr, "Error allocating memory\n");
return;
pc->m<<=1;
}
- pc->chunks[pc->n].from = pmin;
- pc->chunks[pc->n].to = pmax;
+ pc->chunks[pc->n].beg = pmin;
+ pc->chunks[pc->n].end = pmax;
pc->n++;
} else { //the other line, check for overlapping
if ( pmin == -1 && kh_exist(read_pairs, k) ) { //job done, delete entry
int i;
for (i=0; i<pc->n; i++) {
- if ( pmin >= pc->chunks[i].to )
+ if ( pmin >= pc->chunks[i].end )
continue;
- if ( pmax <= pc->chunks[i].from ) //no overlap
+ if ( pmax <= pc->chunks[i].beg ) //no overlap
break;
- if ( pmin < pc->chunks[i].from ) { //overlap at the beginning
- round_buffer_insert_read(&(stats->cov_rbuf), pmin, pc->chunks[i].from-1);
- pmin = pc->chunks[i].from;
+ if ( pmin < pc->chunks[i].beg ) { //overlap at the beginning
+ round_buffer_insert_read(&(stats->cov_rbuf), pmin, pc->chunks[i].beg);
+ pmin = pc->chunks[i].beg;
}
- if ( pmax <= pc->chunks[i].to ) { //completely contained
+ if ( pmax <= pc->chunks[i].end ) { //completely contained
stats->nbases_mapped_cigar -= (pmax - pmin);
return;
} else { //overlap at the end
- stats->nbases_mapped_cigar -= (pc->chunks[i].to - pmin);
- pmin = pc->chunks[i].to;
+ stats->nbases_mapped_cigar -= (pc->chunks[i].end - pmin);
+ pmin = pc->chunks[i].end;
}
}
}
}
- round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax-1);
+ round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
}
void collect_stats(bam1_t *bam_line, stats_t *stats, khash_t(qn2pair) *read_pairs)
stats->nreads_dup++;
}
+ uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
+
int read_len = unclipped_length(bam_line);
if ( read_len >= stats->nbases )
realloc_buffers(stats,read_len);
// Update max_len observed
if ( stats->max_len<read_len )
stats->max_len = read_len;
- if ( IS_READ1(bam_line) && stats->max_len_1st < read_len )
+ if ( order == READ_ORDER_FIRST && stats->max_len_1st < read_len )
stats->max_len_1st = read_len;
- if ( IS_READ2(bam_line) && stats->max_len_2nd < read_len )
+ if ( order == READ_ORDER_LAST && stats->max_len_2nd < read_len )
stats->max_len_2nd = read_len;
int i;
if ( IS_ORIGINAL(bam_line) )
{
stats->read_lengths[read_len]++;
- if ( IS_READ1(bam_line) ) stats->read_lengths_1st[read_len]++;
- if ( IS_READ2(bam_line) ) stats->read_lengths_2nd[read_len]++;
+ if ( order == READ_ORDER_FIRST ) stats->read_lengths_1st[read_len]++;
+ if ( order == READ_ORDER_LAST ) stats->read_lengths_2nd[read_len]++;
collect_orig_read_stats(bam_line, stats, &gc_count);
}
isize = stats->info->nisize;
if ( isize>0 || bam_line->core.tid==bam_line->core.mtid )
{
- int pos_fst = bam_line->core.mpos - bam_line->core.pos;
+ hts_pos_t pos_fst = bam_line->core.mpos - bam_line->core.pos;
int is_fst = IS_READ1(bam_line) ? 1 : -1;
int is_fwd = IS_REVERSE(bam_line) ? -1 : 1;
int is_mfwd = IS_MATE_REVERSE(bam_line) ? -1 : 1;
if ( stats->regions )
{
// Count only on-target bases
- int iref = bam_line->core.pos + 1;
+ hts_pos_t iref = bam_line->core.pos + 1;
for (i=0; i<bam_line->core.n_cigar; i++)
{
int cig = bam_cigar_op(bam_get_cigar(bam_line)[i]);
}
if ( stats->last_pair_tid != bam_line->core.tid) {
- stats->pair_count -= cleanup_overlaps(read_pairs, INT_MAX-1);
+ stats->pair_count -= cleanup_overlaps(read_pairs, INT64_MAX-1);
stats->last_pair_tid = bam_line->core.tid;
stats->last_read_flush = 0;
}
// Coverage distribution graph
round_buffer_flush(stats,bam_line->core.pos);
if ( stats->regions ) {
- uint32_t p = bam_line->core.pos, pnew, pmin, pmax, j;
- pmin = pmax = i = j = 0;
+ hts_pos_t p = bam_line->core.pos, pnew, pmin = 0, pmax = 0;
+ uint32_t j = 0;
+ i = 0;
while ( j < bam_line->core.n_cigar && i < stats->nchunks ) {
int op = bam_cigar_op(bam_get_cigar(bam_line)[j]);
int oplen = bam_cigar_oplen(bam_get_cigar(bam_line)[j]);
case BAM_CMATCH:
case BAM_CEQUAL:
case BAM_CDIFF:
- pmin = MAX(p, stats->chunks[i].from-1);
- pmax = MIN(p+oplen, stats->chunks[i].to);
- if ( pmax >= pmin ) {
+ pmin = MAX(p, stats->chunks[i].beg-1); // 0 based
+ pmax = MIN(p+oplen, stats->chunks[i].end); // 1 based
+ if ( pmax > pmin ) {
if ( stats->info->remove_overlaps )
remove_overlaps(bam_line, read_pairs, stats, pmin, pmax);
else
- round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax-1);
+ round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
}
break;
case BAM_CDEL:
}
pnew = p + (bam_cigar_type(op)&2 ? oplen : 0); // consumes reference
- if ( pnew >= stats->chunks[i].to ) {
+ if ( pnew >= stats->chunks[i].end ) {
// go to the next chunk
i++;
} else {
}
}
} else {
- uint32_t p = bam_line->core.pos, j;
+ hts_pos_t p = bam_line->core.pos;
+ uint32_t j;
for (j = 0; j < bam_line->core.n_cigar; j++) {
int op = bam_cigar_op(bam_get_cigar(bam_line)[j]);
int oplen = bam_cigar_oplen(bam_get_cigar(bam_line)[j]);
if ( stats->info->remove_overlaps )
remove_overlaps(bam_line, read_pairs, stats, p, p+oplen);
else
- round_buffer_insert_read(&(stats->cov_rbuf), p, p+oplen-1);
+ round_buffer_insert_read(&(stats->cov_rbuf), p, p+oplen);
break;
case BAM_CDEL:
break;
}
}
if ( stats->info->remove_overlaps )
- remove_overlaps(bam_line, read_pairs, stats, -1, -1); //remove the line from the hash table
+ remove_overlaps(bam_line, read_pairs, stats, -1LL, -1LL); //remove the line from the hash table
}
}
float n,d;
int k;
- n = p*(N+1)/100;
+ n = (float)p*(N+1)/100;
k = n;
if ( k<=0 )
return gcd[0].depth;
fprintf(to, "# CHK, CRC32 of reads which passed filtering followed by addition (32bit overflow)\n");
fprintf(to, "CHK\t%08x\t%08x\t%08x\n", stats->checksum.names,stats->checksum.reads,stats->checksum.quals);
fprintf(to, "# Summary Numbers. Use `grep ^SN | cut -f 2-` to extract this part.\n");
- fprintf(to, "SN\traw total sequences:\t%ld\n", (long)(stats->nreads_filtered+stats->nreads_1st+stats->nreads_2nd)); // not counting excluded seqs (and none of the below)
+ fprintf(to, "SN\traw total sequences:\t%ld\n", (long)(stats->nreads_filtered+stats->nreads_1st+stats->nreads_2nd+stats->nreads_other)); // not counting excluded seqs (and none of the below)
fprintf(to, "SN\tfiltered sequences:\t%ld\n", (long)stats->nreads_filtered);
- fprintf(to, "SN\tsequences:\t%ld\n", (long)(stats->nreads_1st+stats->nreads_2nd));
+ fprintf(to, "SN\tsequences:\t%ld\n", (long)(stats->nreads_1st+stats->nreads_2nd+stats->nreads_other));
fprintf(to, "SN\tis sorted:\t%d\n", stats->is_sorted ? 1 : 0);
fprintf(to, "SN\t1st fragments:\t%ld\n", (long)stats->nreads_1st);
fprintf(to, "SN\tlast fragments:\t%ld\n", (long)stats->nreads_2nd);
fprintf(to, "SN\tbases duplicated:\t%ld\n", (long)stats->total_len_dup);
fprintf(to, "SN\tmismatches:\t%ld\t# from NM fields\n", (long)stats->nmismatches);
fprintf(to, "SN\terror rate:\t%e\t# mismatches / bases mapped (cigar)\n", stats->nbases_mapped_cigar ? (float)stats->nmismatches/stats->nbases_mapped_cigar : 0);
- float avg_read_length = (stats->nreads_1st+stats->nreads_2nd)?stats->total_len/(stats->nreads_1st+stats->nreads_2nd):0;
+ float avg_read_length = (stats->nreads_1st+stats->nreads_2nd+stats->nreads_other)?stats->total_len/(stats->nreads_1st+stats->nreads_2nd+stats->nreads_other):0;
fprintf(to, "SN\taverage length:\t%.0f\n", avg_read_length);
fprintf(to, "SN\taverage first fragment length:\t%.0f\n", stats->nreads_1st? (float)stats->total_len_1st/stats->nreads_1st:0);
fprintf(to, "SN\taverage last fragment length:\t%.0f\n", stats->nreads_2nd? (float)stats->total_len_2nd/stats->nreads_2nd:0);
fprintf(to, "SN\toutward oriented pairs:\t%ld\n", (long)nisize_outward);
fprintf(to, "SN\tpairs with other orientation:\t%ld\n", (long)nisize_other);
fprintf(to, "SN\tpairs on different chromosomes:\t%ld\n", (long)stats->nreads_anomalous/2);
- fprintf(to, "SN\tpercentage of properly paired reads (%%):\t%.1f\n", (stats->nreads_1st+stats->nreads_2nd)? (float)(100*stats->nreads_properly_paired)/(stats->nreads_1st+stats->nreads_2nd):0);
+ fprintf(to, "SN\tpercentage of properly paired reads (%%):\t%.1f\n", (stats->nreads_1st+stats->nreads_2nd+stats->nreads_other)? (float)(100*stats->nreads_properly_paired)/(stats->nreads_1st+stats->nreads_2nd+stats->nreads_other):0);
if ( stats->target_count ) {
fprintf(to, "SN\tbases inside the target:\t%u\n", stats->target_count);
for (icov=stats->info->cov_threshold+1; icov<stats->ncov; icov++)
100.*(acgtno_count_1st->other + acgtno_count_2nd->other)/acgt_sum);
}
+
+ uint64_t tA=0, tC=0, tG=0, tT=0, tN=0;
fprintf(to, "# ACGT content per cycle for first fragments. Use `grep ^FBC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N and O counts as a percentage of all A/C/G/T bases [%%]\n");
for (ibase=0; ibase<stats->max_len; ibase++)
{
acgtno_count_t *acgtno_count_1st = &(stats->acgtno_cycles_1st[ibase]);
uint64_t acgt_sum_1st = acgtno_count_1st->a + acgtno_count_1st->c + acgtno_count_1st->g + acgtno_count_1st->t;
+ tA += acgtno_count_1st->a;
+ tC += acgtno_count_1st->c;
+ tG += acgtno_count_1st->g;
+ tT += acgtno_count_1st->t;
+ tN += acgtno_count_1st->n;
if ( acgt_sum_1st )
fprintf(to, "FBC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
100.*acgtno_count_1st->other/acgt_sum_1st);
}
+ fprintf(to, "# ACGT raw counters for first fragments. Use `grep ^FTC | cut -f 2-` to extract this part. The columns are: A,C,G,T,N base counters\n");
+ fprintf(to, "FTC\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\n", tA, tC, tG, tT, tN);
+ tA=0, tC=0, tG=0, tT=0, tN=0;
fprintf(to, "# ACGT content per cycle for last fragments. Use `grep ^LBC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N and O counts as a percentage of all A/C/G/T bases [%%]\n");
for (ibase=0; ibase<stats->max_len; ibase++)
{
acgtno_count_t *acgtno_count_2nd = &(stats->acgtno_cycles_2nd[ibase]);
uint64_t acgt_sum_2nd = acgtno_count_2nd->a + acgtno_count_2nd->c + acgtno_count_2nd->g + acgtno_count_2nd->t;
+ tA += acgtno_count_2nd->a;
+ tC += acgtno_count_2nd->c;
+ tG += acgtno_count_2nd->g;
+ tT += acgtno_count_2nd->t;
+ tN += acgtno_count_2nd->n;
if ( acgt_sum_2nd )
fprintf(to, "LBC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
100.*acgtno_count_2nd->other/acgt_sum_2nd);
}
+ fprintf(to, "# ACGT raw counters for last fragments. Use `grep ^LTC | cut -f 2-` to extract this part. The columns are: A,C,G,T,N base counters\n");
+ fprintf(to, "LTC\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\n", tA, tC, tG, tT, tN);
+
+ int tag;
+ for (tag=0; tag<stats->ntags; tag++) {
+ if (stats->tags_barcode[tag].nbases) {
+ fprintf(to, "# ACGT content per cycle for barcodes. Use `grep ^%sC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N counts as a percentage of all A/C/G/T bases [%%]\n",
+ stats->tags_barcode[tag].tag_name);
+ for (ibase=0; ibase<stats->tags_barcode[tag].nbases; ibase++)
+ {
+ if (ibase == stats->tags_barcode[tag].tag_sep)
+ continue;
+
+ acgtno_count_t *acgtno_count = stats->acgtno_barcode + stats->tags_barcode[tag].offset + ibase;
+ uint64_t acgt_sum = acgtno_count->a + acgtno_count->c + acgtno_count->g + acgtno_count->t;
+
+ if ( acgt_sum )
+ fprintf(to, "%sC%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", stats->tags_barcode[tag].tag_name,
+ stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? 1 : 2,
+ stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? ibase+1 : ibase-stats->tags_barcode[tag].tag_sep,
+ 100.*acgtno_count->a/acgt_sum,
+ 100.*acgtno_count->c/acgt_sum,
+ 100.*acgtno_count->g/acgt_sum,
+ 100.*acgtno_count->t/acgt_sum,
+ 100.*acgtno_count->n/acgt_sum);
+ }
+
+ fprintf(to, "# Barcode Qualities. Use `grep ^%sQ | cut -f 2-` to extract this part.\n", stats->tags_barcode[tag].qual_name);
+ fprintf(to, "# Columns correspond to qualities and rows to barcode cycles. First column is the cycle number.\n");
+ for (ibase=0; ibase<stats->tags_barcode[tag].nbases; ibase++)
+ {
+ if (ibase == stats->tags_barcode[tag].tag_sep)
+ continue;
+
+ fprintf(to, "%sQ%d\t%d", stats->tags_barcode[tag].qual_name,
+ stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? 1 : 2,
+ stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? ibase+1 : ibase-stats->tags_barcode[tag].tag_sep);
+ for (iqual=0; iqual<=stats->tags_barcode[tag].max_qual; iqual++)
+ {
+ fprintf(to, "\t%ld", (long)stats->quals_barcode[(stats->tags_barcode[tag].offset + ibase)*stats->nquals+iqual]);
+ }
+ fprintf(to, "\n");
+ }
+ }
+ }
+
fprintf(to, "# Insert sizes. Use `grep ^IS | cut -f 2-` to extract this part. The columns are: insert size, pairs total, inward oriented pairs, outward oriented pairs, other pairs\n");
for (isize=0; isize<ibulk; isize++) {
long in = (long)(stats->isize->inward(stats->isize->data, isize));
}
}
-void init_regions(stats_t *stats, const char *file)
+static void init_regions(stats_t *stats, const char *file)
{
FILE *fp = fopen(file,"r");
if ( !fp ) error("%s: %s\n",file,strerror(errno));
kstring_t line = { 0, 0, NULL };
int warned = 0, r, p, new_p;
- int prev_tid=-1, prev_pos=-1;
+ int prev_tid=-1;
+ hts_pos_t prev_pos=-1LL;
while (line.l = 0, kgetline(&line, (kgets_func *)fgets, fp) >= 0)
{
if ( line.s[0] == '#' ) continue;
if ( tid >= stats->nregions )
{
- stats->regions = realloc(stats->regions,sizeof(regions_t)*(stats->nregions+100));
+ if(!(stats->regions = realloc(stats->regions,sizeof(regions_t)*(tid+REG_INC))))
+ error("Could not allocate memory for region.\n");
+
int j;
- for (j=stats->nregions; j<stats->nregions+100; j++)
+ for (j=stats->nregions; j<tid+REG_INC; j++)
{
stats->regions[j].npos = stats->regions[j].mpos = stats->regions[j].cpos = 0;
stats->regions[j].pos = NULL;
}
- stats->nregions += 100;
+ stats->nregions = tid+REG_INC;
}
int npos = stats->regions[tid].npos;
if ( npos >= stats->regions[tid].mpos )
{
- stats->regions[tid].mpos += 1000;
- stats->regions[tid].pos = realloc(stats->regions[tid].pos,sizeof(pos_t)*stats->regions[tid].mpos);
+ stats->regions[tid].mpos = npos+POS_INC;
+ if (!(stats->regions[tid].pos = realloc(stats->regions[tid].pos, sizeof(hts_pair_pos_t)*stats->regions[tid].mpos)))
+ error("Could not allocate memory for interval.\n");
}
- if ( (sscanf(&line.s[i+1],"%u %u",&stats->regions[tid].pos[npos].from,&stats->regions[tid].pos[npos].to))!=2 ) error("Could not parse the region [%s]\n", &line.s[i+1]);
+ if ( (sscanf(&line.s[i+1],"%"SCNd64" %"SCNd64, &stats->regions[tid].pos[npos].beg, &stats->regions[tid].pos[npos].end))!=2 ) error("Could not parse the region [%s]\n", &line.s[i+1]);
if ( prev_tid==-1 || prev_tid!=tid )
{
prev_tid = tid;
- prev_pos = stats->regions[tid].pos[npos].from;
+ prev_pos = stats->regions[tid].pos[npos].beg;
}
- if ( prev_pos>stats->regions[tid].pos[npos].from )
- error("The positions are not in chromosomal order (%s:%d comes after %d)\n", line.s,stats->regions[tid].pos[npos].from,prev_pos);
+ if ( prev_pos>stats->regions[tid].pos[npos].beg )
+ error("The positions are not in chromosomal order (%s:%"PRIhts_pos" comes after %"PRIhts_pos")\n", line.s, stats->regions[tid].pos[npos].beg, prev_pos);
stats->regions[tid].npos++;
if ( stats->regions[tid].npos > stats->nchunks )
stats->nchunks = stats->regions[tid].npos;
for (r = 0; r < stats->nregions; r++) {
regions_t *reg = &stats->regions[r];
if ( reg->npos > 1 ) {
- qsort(reg->pos, reg->npos, sizeof(pos_t), regions_lt);
+ qsort(reg->pos, reg->npos, sizeof(hts_pair_pos_t), regions_lt);
for (new_p = 0, p = 1; p < reg->npos; p++) {
- if ( reg->pos[new_p].to < reg->pos[p].from )
+ if ( reg->pos[new_p].end < reg->pos[p].beg )
reg->pos[++new_p] = reg->pos[p];
- else if ( reg->pos[new_p].to < reg->pos[p].to )
- reg->pos[new_p].to = reg->pos[p].to;
+ else if ( reg->pos[new_p].end < reg->pos[p].end )
+ reg->pos[new_p].end = reg->pos[p].end;
}
reg->npos = ++new_p;
}
for (p = 0; p < reg->npos; p++)
- stats->target_count += (reg->pos[p].to - reg->pos[p].from + 1);
+ stats->target_count += (reg->pos[p].end - reg->pos[p].beg + 1);
}
- stats->chunks = calloc(stats->nchunks, sizeof(pos_t));
+ if (!(stats->chunks = calloc(stats->nchunks, sizeof(hts_pair_pos_t))))
+ error("Could not allocate memory for chunk.\n");
}
void destroy_regions(stats_t *stats)
// Find a matching interval or skip this read. No splicing of reads is done, no indels or soft clips considered,
// even small overlap is enough to include the read in the stats.
int i = reg->cpos;
- while ( i<reg->npos && reg->pos[i].to<=bam_line->core.pos ) i++;
+ while ( i<reg->npos && reg->pos[i].end<=bam_line->core.pos ) i++;
if ( i>=reg->npos ) { reg->cpos = reg->npos; return 0; }
int64_t endpos = bam_endpos(bam_line);
- if ( endpos < reg->pos[i].from ) return 0;
+ if ( endpos < reg->pos[i].beg ) return 0;
//found a read overlapping a region
reg->cpos = i;
- stats->reg_from = reg->pos[i].from;
- stats->reg_to = reg->pos[i].to;
+ stats->reg_from = reg->pos[i].beg;
+ stats->reg_to = reg->pos[i].end;
//now find all the overlapping chunks
stats->nchunks = 0;
while (i < reg->npos) {
- if (bam_line->core.pos < reg->pos[i].to && endpos >= reg->pos[i].from) {
- stats->chunks[stats->nchunks].from = MAX(bam_line->core.pos+1, reg->pos[i].from);
- stats->chunks[stats->nchunks].to = MIN(endpos, reg->pos[i].to);
+ if (bam_line->core.pos < reg->pos[i].end && endpos >= reg->pos[i].beg) {
+ stats->chunks[stats->nchunks].beg = MAX(bam_line->core.pos+1, reg->pos[i].beg);
+ stats->chunks[stats->nchunks].end = MIN(endpos, reg->pos[i].end);
stats->nchunks++;
}
i++;
int i, j, tid;
stats->nregions = iter->n_reg;
stats->regions = calloc(stats->nregions, sizeof(regions_t));
- stats->chunks = calloc(stats->nchunks, sizeof(pos_t));
+ stats->chunks = calloc(stats->nchunks, sizeof(hts_pair_pos_t));
if ( !stats->regions || !stats->chunks )
return 1;
}
stats->regions[tid].mpos = stats->regions[tid].npos = iter->reg_list[i].count;
- stats->regions[tid].pos = calloc(stats->regions[tid].mpos, sizeof(pos_t));
+ stats->regions[tid].pos = calloc(stats->regions[tid].mpos, sizeof(hts_pair_pos_t));
if ( !stats->regions[tid].pos )
return 1;
for (j = 0; j < stats->regions[tid].npos; j++) {
- stats->regions[tid].pos[j].from = iter->reg_list[i].intervals[j].beg+1;
- stats->regions[tid].pos[j].to = iter->reg_list[i].intervals[j].end;
+ stats->regions[tid].pos[j].beg = iter->reg_list[i].intervals[j].beg+1;
+ stats->regions[tid].pos[j].end = iter->reg_list[i].intervals[j].end;
- stats->target_count += (stats->regions[tid].pos[j].to - stats->regions[tid].pos[j].from + 1);
+ stats->target_count += (stats->regions[tid].pos[j].end - stats->regions[tid].pos[j].beg + 1);
}
}
}
-static void error(const char *format, ...)
+static void HTS_NORETURN error(const char *format, ...)
{
if ( !format )
{
printf("Options:\n");
printf(" -c, --coverage <int>,<int>,<int> Coverage distribution min,max,step [1,1000,1]\n");
printf(" -d, --remove-dups Exclude from statistics reads marked as duplicates\n");
+ printf(" -X, --customized-index-file Use a customized index file\n");
printf(" -f, --required-flag <str|int> Required flag, 0 for unset. See also `samtools flags` [0]\n");
printf(" -F, --filtering-flag <str|int> Filtering flag, 0 for unset. See also `samtools flags` [0]\n");
printf(" --GC-depth <float> the size of GC-depth bins (decreasing bin size increases memory requirement) [2e4]\n");
printf(" -h, --help This help message\n");
printf(" -i, --insert-size <int> Maximum insert size [8000]\n");
printf(" -I, --id <string> Include only listed read group or sample name\n");
- printf(" -l, --read-length <int> Include in the statistics only reads with the given read length []\n");
+ printf(" -l, --read-length <int> Include in the statistics only reads with the given read length [-1]\n");
printf(" -m, --most-inserts <float> Report only the main part of inserts [0.99]\n");
printf(" -P, --split-prefix <str> Path or string prefix for filepaths output by -S (default is input filename)\n");
printf(" -q, --trim-quality <int> The BWA trimming parameter [0]\n");
printf(" -t, --target-regions <file> Do stats in these regions only. Tab-delimited file chr,from,to, 1-based, inclusive.\n");
printf(" -x, --sparse Suppress outputting IS rows where there are no insertions.\n");
printf(" -p, --remove-overlaps Remove overlaps of paired-end reads from coverage and base count computations.\n");
- printf(" -g, --cov-threshold Only bases with coverage above this value will be included in the target percentage computation.\n");
- sam_global_opt_help(stdout, "-.--.@");
+ printf(" -g, --cov-threshold <int> Only bases with coverage above this value will be included in the target percentage computation [0]\n");
+ sam_global_opt_help(stdout, "-.--.@-.");
printf("\n");
}
else
free(stats->ins_cycles_2nd);
free(stats->del_cycles_1st);
free(stats->del_cycles_2nd);
+ if (stats->acgtno_barcode) free(stats->acgtno_barcode);
+ if (stats->quals_barcode) free(stats->quals_barcode);
+ free(stats->tags_barcode);
destroy_regions(stats);
if ( stats->rg_hash ) khash_str2int_destroy(stats->rg_hash);
free(stats->split_name);
void destroy_split_stats(khash_t(c2stats) *split_hash)
{
+ if (!split_hash)
+ return;
+
int i = 0;
stats_t *curr_stats = NULL;
for(i = kh_begin(split_hash); i != kh_end(split_hash); ++i){
stats_info_t* stats_info_init(int argc, char *argv[])
{
stats_info_t* info = calloc(1, sizeof(stats_info_t));
+ if (!info) {
+ return NULL;
+ }
+
info->nisize = 8000;
info->isize_main_bulk = 0.99; // There are always outliers at the far end
info->gcd_bin_size = 20e3;
stats_t* stats_init()
{
stats_t *stats = calloc(1,sizeof(stats_t));
+ if (!stats)
+ return NULL;
+
stats->ngc = 200;
stats->nquals = 256;
stats->nbases = 300;
stats->rseq_pos = -1;
- stats->tid = stats->gcd_pos = -1;
+ stats->tid = -1;
+ stats->gcd_pos = -1LL;
stats->igcd = 0;
stats->is_sorted = 1;
stats->nindels = stats->nbases;
return stats;
}
+static int init_barcode_tags(stats_t* stats) {
+ stats->ntags = 4;
+ stats->tags_barcode = calloc(stats->ntags, sizeof(barcode_info_t));
+ if (!stats->tags_barcode)
+ return -1;
+ stats->tags_barcode[0] = (barcode_info_t){"BC", "QT", 0, -1, -1, 0};
+ stats->tags_barcode[1] = (barcode_info_t){"CR", "CY", 0, -1, -1, 0};
+ stats->tags_barcode[2] = (barcode_info_t){"OX", "BZ", 0, -1, -1, 0};
+ stats->tags_barcode[3] = (barcode_info_t){"RX", "QX", 0, -1, -1, 0};
+ return 0;
+}
+
static void init_stat_structs(stats_t* stats, stats_info_t* info, const char* group_id, const char* targets)
{
// Give stats_t a pointer to the info struct
stats->ncov = 3 + (info->cov_max-info->cov_min) / info->cov_step;
info->cov_max = info->cov_min + ((info->cov_max-info->cov_min)/info->cov_step +1)*info->cov_step - 1;
stats->cov = calloc(sizeof(uint64_t),stats->ncov);
+ if (!stats->cov) goto nomem;
stats->cov_rbuf.size = stats->nbases*5;
stats->cov_rbuf.buffer = calloc(sizeof(int32_t),stats->cov_rbuf.size);
-
+ if (!stats->cov_rbuf.buffer) goto nomem;
if ( group_id ) init_group_id(stats, group_id);
// .. arrays
stats->quals_1st = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
+ if (!stats->quals_1st) goto nomem;
stats->quals_2nd = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
+ if (!stats->quals_2nd) goto nomem;
stats->gc_1st = calloc(stats->ngc,sizeof(uint64_t));
+ if (!stats->gc_1st) goto nomem;
stats->gc_2nd = calloc(stats->ngc,sizeof(uint64_t));
+ if (!stats->gc_2nd) goto nomem;
stats->isize = init_isize_t(info->nisize ?info->nisize+1 :0);
+ if (!stats->isize) goto nomem;
stats->gcd = calloc(stats->ngcd,sizeof(gc_depth_t));
- stats->mpc_buf = info->fai ? calloc(stats->nquals*stats->nbases,sizeof(uint64_t)) : NULL;
+ if (!stats->gcd) goto nomem;
+ if (info->fai) {
+ stats->mpc_buf = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
+ if (!stats->mpc_buf) goto nomem;
+ } else {
+ stats->mpc_buf = NULL;
+ }
stats->acgtno_cycles_1st = calloc(stats->nbases,sizeof(acgtno_count_t));
+ if (!stats->acgtno_cycles_1st) goto nomem;
stats->acgtno_cycles_2nd = calloc(stats->nbases,sizeof(acgtno_count_t));
+ if (!stats->acgtno_cycles_2nd) goto nomem;
stats->read_lengths = calloc(stats->nbases,sizeof(uint64_t));
+ if (!stats->read_lengths) goto nomem;
stats->read_lengths_1st = calloc(stats->nbases,sizeof(uint64_t));
+ if (!stats->read_lengths_1st) goto nomem;
stats->read_lengths_2nd = calloc(stats->nbases,sizeof(uint64_t));
+ if (!stats->read_lengths_2nd) goto nomem;
stats->insertions = calloc(stats->nbases,sizeof(uint64_t));
+ if (!stats->insertions) goto nomem;
stats->deletions = calloc(stats->nbases,sizeof(uint64_t));
+ if (!stats->deletions) goto nomem;
stats->ins_cycles_1st = calloc(stats->nbases+1,sizeof(uint64_t));
+ if (!stats->ins_cycles_1st) goto nomem;
stats->ins_cycles_2nd = calloc(stats->nbases+1,sizeof(uint64_t));
+ if (!stats->ins_cycles_2nd) goto nomem;
stats->del_cycles_1st = calloc(stats->nbases+1,sizeof(uint64_t));
+ if (!stats->del_cycles_1st) goto nomem;
stats->del_cycles_2nd = calloc(stats->nbases+1,sizeof(uint64_t));
+ if (!stats->del_cycles_2nd) goto nomem;
+ if (init_barcode_tags(stats) < 0)
+ goto nomem;
realloc_rseq_buffer(stats);
if ( targets )
init_regions(stats, targets);
+ return;
+ nomem:
+ error("Out of memory");
}
static stats_t* get_curr_split_stats(bam1_t* bam_line, khash_t(c2stats)* split_hash, stats_info_t* info, char* targets)
khiter_t k = kh_get(c2stats, split_hash, split_name);
if(k == kh_end(split_hash)){
curr_stats = stats_init(); // mallocs new instance
+ if (!curr_stats) {
+ error("Couldn't allocate split stats");
+ }
init_stat_structs(curr_stats, info, NULL, targets);
curr_stats->split_name = split_name;
{
char *targets = NULL;
char *bam_fname = NULL;
+ char *bam_idx_fname = NULL;
char *group_id = NULL;
- int sparse = 0;
+ int sparse = 0, has_index_file = 0, ret = 1;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
stats_info_t *info = stats_info_init(argc, argv);
+ if (!info) {
+ fprintf(stderr, "Could not allocate memory for info.\n");
+ return 1;
+ }
static const struct option loptions[] =
{
{"help", no_argument, NULL, 'h'},
{"remove-dups", no_argument, NULL, 'd'},
{"sam", no_argument, NULL, 's'},
+ {"customized-index-file", required_argument, NULL, 'X'},
{"ref-seq", required_argument, NULL, 'r'},
{"coverage", required_argument, NULL, 'c'},
{"read-length", required_argument, NULL, 'l'},
};
int opt;
- while ( (opt=getopt_long(argc,argv,"?hdsxpr:c:l:i:t:m:q:f:F:g:I:1:S:P:@:",loptions,NULL))>0 )
+ while ( (opt=getopt_long(argc,argv,"?hdsXxpr:c:l:i:t:m:q:f:F:g:I:S:P:@:",loptions,NULL))>0 )
{
switch (opt)
{
case 'f': info->flag_require = bam_str2flag(optarg); break;
case 'F': info->flag_filter |= bam_str2flag(optarg); break;
case 'd': info->flag_filter |= BAM_FDUP; break;
+ case 'X': has_index_file = 1; break;
case 's': break;
case 'r': info->fai = fai_load(optarg);
if (info->fai==NULL)
break;
case '?':
case 'h': error(NULL);
+ /* no break */
default:
if (parse_sam_global_opt(opt, optarg, loptions, &ga) != 0)
error("Unknown argument: %s\n", optarg);
break;
}
}
- if ( optind<argc )
- bam_fname = argv[optind++];
+ bam_fname = argv[optind++];
if ( !bam_fname )
{
if ( isatty(STDIN_FILENO) )
free(info);
return 1;
}
+
+ if (has_index_file && !(bam_idx_fname = argv[optind++])) {
+ fprintf(stderr, "No index file provided\n");
+ free(info);
+ return 1;
+ }
+
if (ga.nthreads > 0)
hts_set_threads(info->sam, ga.nthreads);
stats_t *all_stats = stats_init();
+ if (!all_stats) {
+ fprintf(stderr, "Could not allocate memory for stats.\n");
+ cleanup_stats_info(info);
+ return 1;
+ }
stats_t *curr_stats = NULL;
init_stat_structs(all_stats, info, group_id, targets);
// Init
// .. hash
khash_t(c2stats)* split_hash = kh_init(c2stats);
+ if (!split_hash) goto cleanup_all_stats;
khash_t(qn2pair)* read_pairs = kh_init(qn2pair);
+ if (!read_pairs) goto cleanup_split_hash;
// Collect statistics
bam1_t *bam_line = bam_init1();
- if ( optind<argc )
- {
- int filter = 1;
- // Prepare the region hash table for the multi-region iterator
- void *region_hash = bed_hash_regions(NULL, argv, optind, argc, &filter);
- if (region_hash) {
-
- // Collect stats in selected regions only
- hts_idx_t *bam_idx = sam_index_load(info->sam,bam_fname);
- if (bam_idx) {
-
- int regcount = 0;
- hts_reglist_t *reglist = bed_reglist(region_hash, ALL, ®count);
- if (reglist) {
-
- hts_itr_multi_t *iter = sam_itr_regions(bam_idx, info->sam_header, reglist, regcount);
- if (iter) {
-
- if (!targets) {
- all_stats->nchunks = argc-optind;
- if ( replicate_regions(all_stats, iter) )
- fprintf(stderr, "Replications of the regions failed.");
- }
+ if (!bam_line) goto cleanup_read_pairs;
- if ( all_stats->nregions && all_stats->regions ) {
- while (sam_itr_multi_next(info->sam, iter, bam_line) >= 0) {
- if (info->split_tag) {
- curr_stats = get_curr_split_stats(bam_line, split_hash, info, targets);
- collect_stats(bam_line, curr_stats, read_pairs);
- }
- collect_stats(bam_line, all_stats, read_pairs);
- }
+ if (optind < argc) {
+ // Region:interval arguments in the command line
+ hts_idx_t *bam_idx = NULL;
+ if (has_index_file) {
+ bam_idx = sam_index_load2(info->sam, bam_fname, bam_idx_fname);
+ } else {
+ // If an index filename has not been specified, look alongside the alignment file
+ bam_idx = sam_index_load(info->sam, bam_fname);
+ }
+
+ if (bam_idx) {
+ hts_itr_multi_t *iter = sam_itr_regarray(bam_idx, info->sam_header, &argv[optind], argc - optind);
+ if (iter) {
+ if (!targets) {
+ all_stats->nchunks = argc-optind;
+ if (replicate_regions(all_stats, iter))
+ fprintf(stderr, "Replications of the regions failed\n");
+ }
+
+ if ( all_stats->nregions && all_stats->regions ) {
+ while ((ret = sam_itr_next(info->sam, iter, bam_line)) >= 0) {
+ if (info->split_tag) {
+ curr_stats = get_curr_split_stats(bam_line, split_hash, info, targets);
+ collect_stats(bam_line, curr_stats, read_pairs);
}
+ collect_stats(bam_line, all_stats, read_pairs);
+ }
+ if (ret < -1) {
+ fprintf(stderr, "Failure while running the iterator\n");
hts_itr_multi_destroy(iter);
- } else {
- fprintf(stderr, "Creation of the region iterator failed.");
- hts_reglist_free(reglist, regcount);
+ hts_idx_destroy(bam_idx);
+ goto cleanup;
}
- } else {
- fprintf(stderr, "Creation of the region list failed.");
}
-
- hts_idx_destroy(bam_idx);
+ hts_itr_multi_destroy(iter);
} else {
- fprintf(stderr, "Random alignment retrieval only works for indexed BAM files.\n");
+ fprintf(stderr, "Multi-region iterator could not be created\n");
+ hts_idx_destroy(bam_idx);
+ goto cleanup;
}
-
- bed_destroy(region_hash);
+ hts_idx_destroy(bam_idx);
} else {
- fprintf(stderr, "Creation of the region hash table failed.\n");
+ if (has_index_file)
+ fprintf(stderr, "Invalid index file '%s'\n", bam_idx_fname);
+ fprintf(stderr, "Random alignment retrieval only works for indexed files\n");
+ goto cleanup;
}
- }
- else
- {
+ } else {
if ( info->cov_threshold > 0 && !targets ) {
- fprintf(stderr, "Coverage percentage calcuation requires a list of target regions\n");
+ fprintf(stderr, "Coverage percentage calculation requires a list of target regions\n");
goto cleanup;
}
// Stream through the entire BAM ignoring off-target regions if -t is given
- int ret;
while ((ret = sam_read1(info->sam, info->sam_header, bam_line)) >= 0) {
if (info->split_tag) {
curr_stats = get_curr_split_stats(bam_line, split_hash, info, targets);
if (ret < -1) {
fprintf(stderr, "Failure while decoding file\n");
- return 1;
+ goto cleanup;
}
}
if (info->split_tag)
output_split_stats(split_hash, bam_fname, sparse);
+ ret = 0;
cleanup:
bam_destroy1(bam_line);
- bam_hdr_destroy(info->sam_header);
+ sam_hdr_destroy(info->sam_header);
sam_global_args_free(&ga);
+cleanup_read_pairs:
+ cleanup_overlaps(read_pairs, INT64_MAX);
+cleanup_split_hash:
+ destroy_split_stats(split_hash);
+cleanup_all_stats:
cleanup_stats(all_stats);
cleanup_stats_info(info);
- destroy_split_stats(split_hash);
- cleanup_overlaps(read_pairs, INT_MAX);
- return 0;
+ return ret;
}
/* stats.c -- This is the former bamcheck integrated into samtools/htslib.
- Copyright (C) 2012-2015 Genome Research Ltd.
+ Copyright (C) 2012-2019 Genome Research Ltd.
Author: Petr Danecek <pd3@sanger.ac.uk>
Author: Sam Nicholls <sam@samnicholls.net>
#include <string.h>
#include <math.h>
#include <ctype.h>
+#include <inttypes.h>
#include <getopt.h>
#include <errno.h>
#include <assert.h>
#include <htslib/faidx.h>
#include <htslib/sam.h>
#include <htslib/hts.h>
-#include "sam_header.h"
+#include <htslib/hts_defs.h>
#include <htslib/khash_str2int.h>
#include "samtools.h"
#include <htslib/khash.h>
#define BWA_MIN_RDLEN 35
#define DEFAULT_CHUNK_NO 8
#define DEFAULT_PAIR_MAX 10000
+#define ERROR_LIMIT 200
// From the spec
// If 0x4 is set, no assumptions can be made about RNAME, POS, CIGAR, MAPQ, bits 0x2, 0x10, 0x100 and 0x800, and the bit 0x20 of the previous read in the template.
+#define IS_PAIRED(bam) ((bam)->core.flag&BAM_FPAIRED)
#define IS_PAIRED_AND_MAPPED(bam) (((bam)->core.flag&BAM_FPAIRED) && !((bam)->core.flag&BAM_FUNMAP) && !((bam)->core.flag&BAM_FMUNMAP))
#define IS_PROPERLYPAIRED(bam) (((bam)->core.flag&(BAM_FPAIRED|BAM_FPROPER_PAIR)) == (BAM_FPAIRED|BAM_FPROPER_PAIR) && !((bam)->core.flag&BAM_FUNMAP))
#define IS_UNMAPPED(bam) ((bam)->core.flag&BAM_FUNMAP)
#define IS_DUP(bam) ((bam)->core.flag&BAM_FDUP)
#define IS_ORIGINAL(bam) (((bam)->core.flag&(BAM_FSECONDARY|BAM_FSUPPLEMENTARY)) == 0)
+#define READ_ORDER_NONE 0
+#define READ_ORDER_FIRST 1
+#define READ_ORDER_LAST 2
+#define READ_ORDER_MIDDLE 3
+
+#define REG_INC 100
+#define POS_INC 1000
+
// The GC-depth graph works as follows: split the reference sequence into
// segments and calculate GC content and depth in each bin. Then sort
// these segments by their GC and plot the depth distribution by means
// For coverage distribution, a simple pileup
typedef struct
{
- int64_t pos;
+ hts_pos_t pos;
int size, start;
int *buffer;
}
round_buffer_t;
-typedef struct { uint32_t from, to; } pos_t;
typedef struct
{
- int npos,mpos,cpos;
- pos_t *pos;
+ int npos, mpos, cpos;
+ hts_pair_pos_t *pos;
}
regions_t;
}
acgtno_count_t;
+typedef struct
+{
+ char tag_name[3];
+ char qual_name[3];
+ uint32_t nbases;
+ int32_t tag_sep; // Index of the separator (if present)
+ int32_t max_qual;
+ uint32_t offset; // Where the tag stats info is located in the allocated memory
+}
+barcode_info_t;
+
typedef struct
{
// Auxiliary data
float isize_main_bulk; // There are always some unrealistically big insert sizes, report only the main part
int cov_min,cov_max,cov_step; // Minimum, maximum coverage and size of the coverage bins
samFile* sam;
- bam_hdr_t* sam_header;
+ sam_hdr_t* sam_header;
// Filters
int filter_readlen;
uint64_t total_len_dup;
uint64_t nreads_1st;
uint64_t nreads_2nd;
+ uint64_t nreads_other;
uint64_t nreads_filtered;
uint64_t nreads_dup;
uint64_t nreads_unmapped;
// GC-depth related data
uint32_t ngcd, igcd; // The maximum number of GC depth bins and index of the current bin
gc_depth_t *gcd; // The GC-depth bins holder
- int32_t tid, gcd_pos; // Position of the current bin
- int32_t pos; // Position of the last read
+ int32_t tid; // Position of the current bin
+ hts_pos_t gcd_pos, pos; // Position of the last read
// Coverage distribution related data
int ncov; // The number of coverage bins
// Mismatches by read cycle
uint8_t *rseq_buf; // A buffer for reference sequence to check the mismatches against
int mrseq_buf; // The size of the buffer
- int32_t rseq_pos; // The coordinate of the first base in the buffer
- int32_t nrseq_buf; // The used part of the buffer
+ hts_pos_t rseq_pos; // The coordinate of the first base in the buffer
+ int64_t nrseq_buf; // The used part of the buffer
uint64_t *mpc_buf; // Mismatches per cycle
// Target regions
- int nregions, reg_from, reg_to;
+ int nregions;
+ hts_pos_t reg_from, reg_to;
regions_t *regions;
// Auxiliary data
char* split_name;
stats_info_t* info; // Pointer to options and settings struct
- pos_t *chunks;
+ hts_pair_pos_t *chunks;
uint32_t nchunks;
uint32_t pair_count; // Number of active pairs in the pairing hash table
uint32_t target_count; // Number of bases covered by the target file
uint32_t last_pair_tid;
uint32_t last_read_flush;
+
+ // Barcode statistics
+ acgtno_count_t *acgtno_barcode;
+ uint64_t *quals_barcode;
+ barcode_info_t *tags_barcode;
+ uint32_t ntags;
+ uint32_t error_number;
}
stats_t;
KHASH_MAP_INIT_STR(c2stats, stats_t*)
typedef struct {
uint32_t first; // 1 - first read, 2 - second read
uint32_t n, m; // number of chunks, allocated chunks
- pos_t *chunks; // chunk array of size m
+ hts_pair_pos_t *chunks; // chunk array of size m
} pair_t;
KHASH_MAP_INIT_STR(qn2pair, pair_t*)
-static void error(const char *format, ...);
+static void HTS_NORETURN error(const char *format, ...);
int is_in_regions(bam1_t *bam_line, stats_t *stats);
void realloc_buffers(stats_t *stats, int seq_len);
static int regions_lt(const void *r1, const void *r2) {
- int64_t from_diff = (int64_t)((pos_t *)r1)->from - (int64_t)((pos_t *)r2)->from;
- int64_t to_diff = (int64_t)((pos_t *)r1)->to - (int64_t)((pos_t *)r2)->to;
+ int64_t from_diff = ((hts_pair_pos_t *)r1)->beg - ((hts_pair_pos_t *)r2)->beg;
+ int64_t to_diff = ((hts_pair_pos_t *)r1)->end - ((hts_pair_pos_t *)r2)->end;
return from_diff > 0 ? 1 : from_diff < 0 ? -1 : to_diff > 0 ? 1 : to_diff < 0 ? -1 : 0;
}
return 1 + (depth - min) / step;
}
-static inline int round_buffer_lidx2ridx(int offset, int size, int64_t refpos, int64_t pos)
+static inline int round_buffer_lidx2ridx(int offset, int size, hts_pos_t refpos, hts_pos_t pos)
{
return (offset + (pos-refpos) % size) % size;
}
-void round_buffer_flush(stats_t *stats, int64_t pos)
+void round_buffer_flush(stats_t *stats, hts_pos_t pos)
{
int ibuf,idp;
if ( pos==stats->cov_rbuf.pos )
return;
- int64_t new_pos = pos;
+ hts_pos_t new_pos = pos;
if ( pos==-1 || pos - stats->cov_rbuf.pos >= stats->cov_rbuf.size )
{
// Flush the whole buffer, but in sequential order,
}
if ( pos < stats->cov_rbuf.pos )
- error("Expected coordinates in ascending order, got %ld after %ld\n", pos,stats->cov_rbuf.pos);
+ error("Expected coordinates in ascending order, got %"PRIhts_pos" after %"PRIhts_pos"\n", pos, stats->cov_rbuf.pos);
int ifrom = stats->cov_rbuf.start;
- int ito = round_buffer_lidx2ridx(stats->cov_rbuf.start,stats->cov_rbuf.size,stats->cov_rbuf.pos,pos-1);
+ int ito = round_buffer_lidx2ridx(stats->cov_rbuf.start, stats->cov_rbuf.size, stats->cov_rbuf.pos, pos-1);
if ( ifrom>ito )
{
for (ibuf=ifrom; ibuf<stats->cov_rbuf.size; ibuf++)
stats->cov[idp]++;
stats->cov_rbuf.buffer[ibuf] = 0;
}
- stats->cov_rbuf.start = (new_pos==-1) ? 0 : round_buffer_lidx2ridx(stats->cov_rbuf.start,stats->cov_rbuf.size,stats->cov_rbuf.pos,pos);
+ stats->cov_rbuf.start = (new_pos==-1) ? 0 : round_buffer_lidx2ridx(stats->cov_rbuf.start, stats->cov_rbuf.size, stats->cov_rbuf.pos, pos);
stats->cov_rbuf.pos = new_pos;
}
-void round_buffer_insert_read(round_buffer_t *rbuf, int64_t from, int64_t to)
+/**
+ * [from, to) - 0 based half-open
+ */
+static void round_buffer_insert_read(round_buffer_t *rbuf, hts_pos_t from, hts_pos_t to)
{
- if ( to-from >= rbuf->size )
- error("The read length too big (%d), please increase the buffer length (currently %d)\n", to-from+1,rbuf->size);
+ if ( to-from > rbuf->size )
+ error("The read length too big (%"PRIhts_pos"), please increase the buffer length (currently %d)\n", to-from, rbuf->size);
if ( from < rbuf->pos )
- error("The reads are not sorted (%ld comes after %ld).\n", from,rbuf->pos);
+ error("The reads are not sorted (%"PRIhts_pos" comes after %"PRIhts_pos").\n", from, rbuf->pos);
- int ifrom,ito,ibuf;
- ifrom = round_buffer_lidx2ridx(rbuf->start,rbuf->size,rbuf->pos,from);
- ito = round_buffer_lidx2ridx(rbuf->start,rbuf->size,rbuf->pos,to);
+ int ifrom, ito, ibuf;
+ ifrom = round_buffer_lidx2ridx(rbuf->start, rbuf->size, rbuf->pos, from);
+ ito = round_buffer_lidx2ridx(rbuf->start, rbuf->size, rbuf->pos, to);
if ( ifrom>ito )
{
for (ibuf=ifrom; ibuf<rbuf->size; ibuf++)
rbuf->buffer[ibuf]++;
ifrom = 0;
}
- for (ibuf=ifrom; ibuf<=ito; ibuf++)
+ for (ibuf=ifrom; ibuf<ito; ibuf++)
rbuf->buffer[ibuf]++;
}
void count_indels(stats_t *stats,bam1_t *bam_line)
{
int is_fwd = IS_REVERSE(bam_line) ? 0 : 1;
- int is_1st = IS_READ1(bam_line) ? 1 : 0;
+ uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
int icig;
int icycle = 0;
int read_len = bam_line->core.l_qseq;
int idx = is_fwd ? icycle : read_len-icycle-ncig;
if ( idx<0 )
error("FIXME: read_len=%d vs icycle=%d\n", read_len,icycle);
- if ( idx >= stats->nbases || idx<0 ) error("FIXME: %d vs %d, %s:%d %s\n", idx,stats->nbases, stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam_get_qname(bam_line));
- if ( is_1st )
+ if ( idx >= stats->nbases || idx<0 ) error("FIXME: %d vs %d, %s:%"PRIhts_pos" %s\n", idx, stats->nbases, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
+ if ( order == READ_ORDER_FIRST )
stats->ins_cycles_1st[idx]++;
- else
+ if ( order == READ_ORDER_LAST )
stats->ins_cycles_2nd[idx]++;
icycle += ncig;
if ( ncig<=stats->nindels )
int idx = is_fwd ? icycle-1 : read_len-icycle-1;
if ( idx<0 ) continue; // discard meaningless deletions
if ( idx >= stats->nbases ) error("FIXME: %d vs %d\n", idx,stats->nbases);
- if ( is_1st )
+ if ( order == READ_ORDER_FIRST )
stats->del_cycles_1st[idx]++;
- else
+ if ( order == READ_ORDER_LAST )
stats->del_cycles_2nd[idx]++;
if ( ncig<=stats->nindels )
stats->deletions[ncig-1]++;
void count_mismatches_per_cycle(stats_t *stats, bam1_t *bam_line, int read_len)
{
int is_fwd = IS_REVERSE(bam_line) ? 0 : 1;
- int icig,iread=0,icycle=0;
- int iref = bam_line->core.pos - stats->rseq_pos;
+ int icig, iread=0, icycle=0;
+ hts_pos_t iref = bam_line->core.pos - stats->rseq_pos;
uint8_t *read = bam_get_seq(bam_line);
uint8_t *quals = bam_get_qual(bam_line);
uint64_t *mpc_buf = stats->mpc_buf;
continue;
}
// Ignore H and N CIGARs. The letter are inserted e.g. by TopHat and often require very large
- // chunk of refseq in memory. Not very frequent and not noticable in the stats.
+ // chunk of refseq in memory. Not very frequent and not noticeable in the stats.
if ( cig==BAM_CREF_SKIP || cig==BAM_CHARD_CLIP || cig==BAM_CPAD ) continue;
if ( cig!=BAM_CMATCH && cig!=BAM_CEQUAL && cig!=BAM_CDIFF ) // not relying on precalculated diffs
- error("TODO: cigar %d, %s:%d %s\n", cig,stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam_get_qname(bam_line));
+ error("TODO: cigar %d, %s:%"PRIhts_pos" %s\n", cig, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
if ( ncig+iref > stats->nrseq_buf )
- error("FIXME: %d+%d > %d, %s, %s:%d\n",ncig,iref,stats->nrseq_buf, bam_get_qname(bam_line),stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1);
+ error("FIXME: %d+%"PRIhts_pos" > %"PRId64", %s, %s:%"PRIhts_pos"\n", ncig, iref, stats->nrseq_buf, bam_get_qname(bam_line), sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1);
int im;
for (im=0; im<ncig; im++)
{
uint8_t qual = quals[iread] + 1;
if ( qual>=stats->nquals )
- error("TODO: quality too high %d>=%d (%s %d %s)\n", qual,stats->nquals, stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam_get_qname(bam_line));
+ error("TODO: quality too high %d>=%d (%s %"PRIhts_pos" %s)\n", qual, stats->nquals, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
int idx = is_fwd ? icycle : read_len-icycle-1;
if ( idx>stats->max_len )
- error("mpc: %d>%d (%s %d %s)\n",idx,stats->max_len,stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam_get_qname(bam_line));
+ error("mpc: %d>%d (%s %"PRIhts_pos" %s)\n", idx, stats->max_len, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
idx = idx*stats->nquals + qual;
if ( idx>=stats->nquals*stats->nbases )
}
}
-void read_ref_seq(stats_t *stats, int32_t tid, int32_t pos)
+void read_ref_seq(stats_t *stats, int32_t tid, hts_pos_t pos)
{
- int i, fai_ref_len;
- char *fai_ref = faidx_fetch_seq(stats->info->fai, stats->info->sam_header->target_name[tid], pos, pos+stats->mrseq_buf-1, &fai_ref_len);
- if ( fai_ref_len<0 ) error("Failed to fetch the sequence \"%s\"\n", stats->info->sam_header->target_name[tid]);
+ int i;
+ hts_pos_t fai_ref_len;
+ char *fai_ref = faidx_fetch_seq64(stats->info->fai, sam_hdr_tid2name(stats->info->sam_header, tid), pos, pos+stats->mrseq_buf-1, &fai_ref_len);
+ if ( fai_ref_len < 0 ) error("Failed to fetch the sequence \"%s\"\n", sam_hdr_tid2name(stats->info->sam_header, tid));
uint8_t *ptr = stats->rseq_buf;
for (i=0; i<fai_ref_len; i++)
stats->tid = tid;
}
-float fai_gc_content(stats_t *stats, int pos, int len)
+float fai_gc_content(stats_t *stats, hts_pos_t pos, int len)
{
uint32_t gc,count,c;
- int i = pos - stats->rseq_pos, ito = i + len;
+ hts_pos_t i = pos - stats->rseq_pos, ito = i + len;
assert( i>=0 );
if ( ito > stats->nrseq_buf ) ito = stats->nrseq_buf;
if ( stats->mrseq_buf<n )
{
stats->rseq_buf = realloc(stats->rseq_buf,sizeof(uint8_t)*n);
+ if (!stats->rseq_buf) {
+ error("Could not reallocate reference sequence buffer");
+ }
stats->mrseq_buf = n;
}
}
// Realloc the coverage distribution buffer
int *rbuffer = calloc(sizeof(int),seq_len*5);
+ if (!rbuffer) {
+ error("Could not allocate coverage distribution buffer");
+ }
n = stats->cov_rbuf.size-stats->cov_rbuf.start;
memcpy(rbuffer,stats->cov_rbuf.buffer+stats->cov_rbuf.start,n);
if ( stats->cov_rbuf.start>1 )
stats->checksum.quals += crc32(0L, qual, (seq_len+1)/2);
}
+// Collect statistics about the barcode tags specified by init_barcode_tags method
+static void collect_barcode_stats(bam1_t* bam_line, stats_t* stats) {
+ uint32_t nbases, tag, i;
+ acgtno_count_t *acgtno;
+ uint64_t *quals;
+ int32_t *separator, *maxqual;
+
+ for (tag = 0; tag < stats->ntags; tag++) {
+ const char *barcode_tag = stats->tags_barcode[tag].tag_name, *qual_tag = stats->tags_barcode[tag].qual_name;
+ uint8_t* bc = bam_aux_get(bam_line, barcode_tag);
+ if (!bc)
+ continue;
+
+ char* barcode = bam_aux2Z(bc);
+ if (!barcode)
+ continue;
+
+ uint32_t barcode_len = strlen(barcode);
+ if (!stats->tags_barcode[tag].nbases) { // tag seen for the first time
+ uint32_t offset = 0;
+ for (i = 0; i < stats->ntags; i++)
+ offset += stats->tags_barcode[i].nbases;
+
+ stats->tags_barcode[tag].offset = offset;
+ stats->tags_barcode[tag].nbases = barcode_len;
+ stats->acgtno_barcode = realloc(stats->acgtno_barcode, (offset + barcode_len) * sizeof(acgtno_count_t));
+ stats->quals_barcode = realloc(stats->quals_barcode, (offset + barcode_len) * stats->nquals * sizeof(uint64_t));
+
+ if (!stats->acgtno_barcode || !stats->quals_barcode)
+ error("Error allocating memory. Aborting!\n");
+
+ memset(stats->acgtno_barcode + offset, 0, barcode_len*sizeof(acgtno_count_t));
+ memset(stats->quals_barcode + offset*stats->nquals, 0, barcode_len*stats->nquals*sizeof(uint64_t));
+ }
+
+ nbases = stats->tags_barcode[tag].nbases;
+ if (barcode_len > nbases) {
+ fprintf(samtools_stderr, "Barcodes with tag %s differ in length at sequence '%s'\n", barcode_tag, bam_get_qname(bam_line));
+ continue;
+ }
+
+ acgtno = stats->acgtno_barcode + stats->tags_barcode[tag].offset;
+ quals = stats->quals_barcode + stats->tags_barcode[tag].offset*stats->nquals;
+ maxqual = &stats->tags_barcode[tag].max_qual;
+ separator = &stats->tags_barcode[tag].tag_sep;
+ int error_flag = 0;
+
+ for (i = 0; i < barcode_len; i++) {
+ switch (barcode[i]) {
+ case 'A':
+ acgtno[i].a++;
+ break;
+ case 'C':
+ acgtno[i].c++;
+ break;
+ case 'G':
+ acgtno[i].g++;
+ break;
+ case 'T':
+ acgtno[i].t++;
+ break;
+ case 'N':
+ acgtno[i].n++;
+ break;
+ default:
+ if (*separator >= 0) {
+ if (*separator != i) {
+ if (stats->error_number < ERROR_LIMIT) {
+ fprintf(samtools_stderr, "Barcode separator for tag %s is in a different position or wrong barcode content('%s') at sequence '%s'\n", barcode_tag, barcode, bam_get_qname(bam_line));
+ stats->error_number++;
+ }
+ error_flag = 1;
+ }
+ } else {
+ *separator = i;
+ }
+ }
+
+ /* don't process the rest of the tag bases */
+ if (error_flag)
+ break;
+ }
+
+ /* skip to the next tag */
+ if (error_flag)
+ continue;
+
+ uint8_t* qt = bam_aux_get(bam_line, qual_tag);
+ if (!qt)
+ continue;
+
+ char* barqual = bam_aux2Z(qt);
+ if (!barqual)
+ continue;
+
+ uint32_t barqual_len = strlen(barqual);
+ if (barqual_len == barcode_len) {
+ for (i = 0; i < barcode_len; i++) {
+ int32_t qual = (int32_t)barqual[i] - '!'; // Phred + 33
+ if (qual >= 0 && qual < stats->nquals) {
+ quals[i * stats->nquals + qual]++;
+ if (qual > *maxqual)
+ *maxqual = qual;
+ }
+ }
+ } else {
+ if (stats->error_number++ < ERROR_LIMIT) {
+ fprintf(samtools_stderr, "%s length and %s length don't match for sequence '%s'\n", barcode_tag, qual_tag, bam_get_qname(bam_line));
+ }
+ }
+ }
+}
+
// These stats should only be calculated for the original reads ignoring
// supplementary artificial reads otherwise we'll accidentally double count
void collect_orig_read_stats(bam1_t *bam_line, stats_t *stats, int* gc_count_out)
if ( bam_line->core.flag & BAM_FQCFAIL ) stats->nreads_QCfailed++;
if ( bam_line->core.flag & BAM_FPAIRED ) stats->nreads_paired_tech++;
+ uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
+
// Count GC and ACGT per cycle. Note that cycle is approximate, clipping is ignored
uint8_t *seq = bam_get_seq(bam_line);
- int i, read_cycle, gc_count = 0, reverse = IS_REVERSE(bam_line), is_first = IS_READ1(bam_line);
- for (i=0; i<seq_len; i++)
- {
- // Read cycle for current index
- read_cycle = (reverse ? seq_len-i-1 : i);
+ int i, read_cycle, gc_count = 0, reverse = IS_REVERSE(bam_line);
- // Conversion from uint8_t coding:
- // -12-4---8------5
- // =ACMGRSVTWYHKDBN
- switch (bam_seqi(seq, i)) {
- case 1:
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].a++ : stats->acgtno_cycles_2nd[ read_cycle ].a++;
- break;
- case 2:
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].c++ : stats->acgtno_cycles_2nd[ read_cycle ].c++;
- gc_count++;
- break;
- case 4:
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].g++ : stats->acgtno_cycles_2nd[ read_cycle ].g++;
- gc_count++;
- break;
- case 8:
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].t++ : stats->acgtno_cycles_2nd[ read_cycle ].t++;
- break;
- case 15:
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].n++ : stats->acgtno_cycles_2nd[ read_cycle ].n++;
- break;
- default:
- /*
- * count "=" sequences in "other" along
- * with MRSVWYHKDB ambiguity codes
- */
- is_first ? stats->acgtno_cycles_1st[ read_cycle ].other++ : stats->acgtno_cycles_2nd[ read_cycle ].other++;
- break;
+ acgtno_count_t *acgtno_cycles = (order == READ_ORDER_FIRST) ? stats->acgtno_cycles_1st : (order == READ_ORDER_LAST) ? stats->acgtno_cycles_2nd : NULL ;
+ if (acgtno_cycles) {
+ for (i=0; i<seq_len; i++)
+ {
+ // Read cycle for current index
+ read_cycle = (reverse ? seq_len-i-1 : i);
+
+ // Conversion from uint8_t coding:
+ // -12-4---8------5
+ // =ACMGRSVTWYHKDBN
+ switch (bam_seqi(seq, i)) {
+ case 1:
+ acgtno_cycles[ read_cycle ].a++;
+ break;
+ case 2:
+ acgtno_cycles[ read_cycle ].c++;
+ gc_count++;
+ break;
+ case 4:
+ acgtno_cycles[ read_cycle ].g++;
+ gc_count++;
+ break;
+ case 8:
+ acgtno_cycles[ read_cycle ].t++;
+ break;
+ case 15:
+ acgtno_cycles[ read_cycle ].n++;
+ break;
+ default:
+ /*
+ * count "=" sequences in "other" along
+ * with MRSVWYHKDB ambiguity codes
+ */
+ acgtno_cycles[ read_cycle ].other++;
+ break;
+ }
}
}
int gc_idx_min = gc_count*(stats->ngc-1)/seq_len;
// Determine which array (1st or 2nd read) will these stats go to,
// trim low quality bases from end the same way BWA does,
// fill GC histogram
- uint64_t *quals;
+ uint64_t *quals = NULL;
uint8_t *bam_quals = bam_get_qual(bam_line);
- if ( IS_READ2(bam_line) )
- {
- quals = stats->quals_2nd;
- stats->nreads_2nd++;
- stats->total_len_2nd += seq_len;
- for (i=gc_idx_min; i<gc_idx_max; i++)
- stats->gc_2nd[i]++;
- }
- else
- {
+
+ switch (order) {
+ case READ_ORDER_FIRST:
quals = stats->quals_1st;
stats->nreads_1st++;
stats->total_len_1st += seq_len;
for (i=gc_idx_min; i<gc_idx_max; i++)
stats->gc_1st[i]++;
+ break;
+ case READ_ORDER_LAST:
+ quals = stats->quals_2nd;
+ stats->nreads_2nd++;
+ stats->total_len_2nd += seq_len;
+ for (i=gc_idx_min; i<gc_idx_max; i++)
+ stats->gc_2nd[i]++;
+ break;
+ default:
+ stats->nreads_other++;
}
if ( stats->info->trim_qual>0 )
stats->nbases_trimmed += bwa_trim_read(stats->info->trim_qual, bam_quals, seq_len, reverse);
// Quality histogram and average quality. Clipping is neglected.
- for (i=0; i<seq_len; i++)
- {
- uint8_t qual = bam_quals[ reverse ? seq_len-i-1 : i];
- if ( qual>=stats->nquals )
- error("TODO: quality too high %d>=%d (%s %d %s)\n", qual,stats->nquals,stats->info->sam_header->target_name[bam_line->core.tid],bam_line->core.pos+1,bam_get_qname(bam_line));
- if ( qual>stats->max_qual )
- stats->max_qual = qual;
+ if (quals) {
+ for (i=0; i<seq_len; i++)
+ {
+ uint8_t qual = bam_quals[ reverse ? seq_len-i-1 : i];
+ if ( qual>=stats->nquals )
+ error("TODO: quality too high %d>=%d (%s %"PRIhts_pos" %s)\n", qual, stats->nquals, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
+ if ( qual>stats->max_qual )
+ stats->max_qual = qual;
+
+ quals[ i*stats->nquals+qual ]++;
+ stats->sum_qual += qual;
+ }
+ }
- quals[ i*stats->nquals+qual ]++;
- stats->sum_qual += qual;
+ // Barcode statistics
+ if (order == READ_ORDER_FIRST) {
+ collect_barcode_stats(bam_line, stats);
}
// Look at the flags and increment appropriate counters (mapped, paired, etc)
*gc_count_out = gc_count;
}
-static int cleanup_overlaps(khash_t(qn2pair) *read_pairs, int max) {
+static int cleanup_overlaps(khash_t(qn2pair) *read_pairs, hts_pos_t max) {
if ( !read_pairs )
return 0;
char *key = (char *)kh_key(read_pairs, k);
pair_t *val = kh_val(read_pairs, k);
if ( val && val->chunks ) {
- if ( val->chunks[val->n-1].to < max ) {
+ if ( val->chunks[val->n-1].end < max ) {
free(val->chunks);
free(val);
free(key);
}
}
}
- if ( max == INT_MAX )
+ if ( max == INT64_MAX )
kh_destroy(qn2pair, read_pairs);
return count;
}
-static void remove_overlaps(bam1_t *bam_line, khash_t(qn2pair) *read_pairs, stats_t *stats, int pmin, int pmax) {
+/**
+ * [pmin, pmax) - 0 based half-open
+ */
+static void remove_overlaps(bam1_t *bam_line, khash_t(qn2pair) *read_pairs, stats_t *stats, hts_pos_t pmin, hts_pos_t pmax) {
if ( !bam_line || !read_pairs || !stats )
return;
- uint32_t first = (IS_READ1(bam_line) > 0 ? 1 : 0) + (IS_READ2(bam_line) > 0 ? 2 : 0) ;
+ uint32_t order = (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0);
if ( !(bam_line->core.flag & BAM_FPAIRED) ||
(bam_line->core.flag & BAM_FMUNMAP) ||
- (abs(bam_line->core.isize) >= 2*bam_line->core.l_qseq) ||
- (first != 1 && first != 2) ) {
+ (llabs(bam_line->core.isize) >= 2*bam_line->core.l_qseq) ||
+ (order != READ_ORDER_FIRST && order != READ_ORDER_LAST) ) {
if ( pmin >= 0 )
- round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax-1);
+ round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
return;
}
char *qname = bam_get_qname(bam_line);
if ( !qname ) {
- fprintf(samtools_stderr, "Error retrieving qname for line starting at pos %d\n", bam_line->core.pos);
+ fprintf(samtools_stderr, "Error retrieving qname for line starting at pos %"PRIhts_pos"\n", bam_line->core.pos);
return;
}
k = kh_put(qn2pair, read_pairs, s, &ret);
if ( -1 == ret ) {
- fprintf(samtools_stderr, "Error inserting read '%s' in pair hash table\n", qname);
- return;
+ error("Error inserting read '%s' in pair hash table\n", qname);
}
pair_t *pc = calloc(1, sizeof(pair_t));
}
pc->m = DEFAULT_CHUNK_NO;
- pc->chunks = calloc(pc->m, sizeof(pos_t));
+ pc->chunks = calloc(pc->m, sizeof(hts_pair_pos_t));
if ( !pc->chunks ) {
fprintf(samtools_stderr, "Error allocating memory\n");
return;
}
- pc->chunks[0].from = pmin;
- pc->chunks[0].to = pmax;
+ pc->chunks[0].beg = pmin;
+ pc->chunks[0].end = pmax;
pc->n = 1;
- pc->first = first;
+ pc->first = order;
kh_val(read_pairs, k) = pc;
stats->pair_count++;
return;
}
- if ( first == pc->first ) { //chunk from an existing line
+ if ( order == pc->first ) { //chunk from an existing line
if ( pmin == -1 )
return;
if ( pc->n == pc->m ) {
- pos_t *tmp = realloc(pc->chunks, (pc->m<<1)*sizeof(pos_t));
+ hts_pair_pos_t *tmp = realloc(pc->chunks, (pc->m<<1)*sizeof(hts_pair_pos_t));
if ( !tmp ) {
fprintf(samtools_stderr, "Error allocating memory\n");
return;
pc->m<<=1;
}
- pc->chunks[pc->n].from = pmin;
- pc->chunks[pc->n].to = pmax;
+ pc->chunks[pc->n].beg = pmin;
+ pc->chunks[pc->n].end = pmax;
pc->n++;
} else { //the other line, check for overlapping
if ( pmin == -1 && kh_exist(read_pairs, k) ) { //job done, delete entry
int i;
for (i=0; i<pc->n; i++) {
- if ( pmin >= pc->chunks[i].to )
+ if ( pmin >= pc->chunks[i].end )
continue;
- if ( pmax <= pc->chunks[i].from ) //no overlap
+ if ( pmax <= pc->chunks[i].beg ) //no overlap
break;
- if ( pmin < pc->chunks[i].from ) { //overlap at the beginning
- round_buffer_insert_read(&(stats->cov_rbuf), pmin, pc->chunks[i].from-1);
- pmin = pc->chunks[i].from;
+ if ( pmin < pc->chunks[i].beg ) { //overlap at the beginning
+ round_buffer_insert_read(&(stats->cov_rbuf), pmin, pc->chunks[i].beg);
+ pmin = pc->chunks[i].beg;
}
- if ( pmax <= pc->chunks[i].to ) { //completely contained
+ if ( pmax <= pc->chunks[i].end ) { //completely contained
stats->nbases_mapped_cigar -= (pmax - pmin);
return;
} else { //overlap at the end
- stats->nbases_mapped_cigar -= (pc->chunks[i].to - pmin);
- pmin = pc->chunks[i].to;
+ stats->nbases_mapped_cigar -= (pc->chunks[i].end - pmin);
+ pmin = pc->chunks[i].end;
}
}
}
}
- round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax-1);
+ round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
}
void collect_stats(bam1_t *bam_line, stats_t *stats, khash_t(qn2pair) *read_pairs)
stats->nreads_dup++;
}
+ uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
+
int read_len = unclipped_length(bam_line);
if ( read_len >= stats->nbases )
realloc_buffers(stats,read_len);
// Update max_len observed
if ( stats->max_len<read_len )
stats->max_len = read_len;
- if ( IS_READ1(bam_line) && stats->max_len_1st < read_len )
+ if ( order == READ_ORDER_FIRST && stats->max_len_1st < read_len )
stats->max_len_1st = read_len;
- if ( IS_READ2(bam_line) && stats->max_len_2nd < read_len )
+ if ( order == READ_ORDER_LAST && stats->max_len_2nd < read_len )
stats->max_len_2nd = read_len;
int i;
if ( IS_ORIGINAL(bam_line) )
{
stats->read_lengths[read_len]++;
- if ( IS_READ1(bam_line) ) stats->read_lengths_1st[read_len]++;
- if ( IS_READ2(bam_line) ) stats->read_lengths_2nd[read_len]++;
+ if ( order == READ_ORDER_FIRST ) stats->read_lengths_1st[read_len]++;
+ if ( order == READ_ORDER_LAST ) stats->read_lengths_2nd[read_len]++;
collect_orig_read_stats(bam_line, stats, &gc_count);
}
isize = stats->info->nisize;
if ( isize>0 || bam_line->core.tid==bam_line->core.mtid )
{
- int pos_fst = bam_line->core.mpos - bam_line->core.pos;
+ hts_pos_t pos_fst = bam_line->core.mpos - bam_line->core.pos;
int is_fst = IS_READ1(bam_line) ? 1 : -1;
int is_fwd = IS_REVERSE(bam_line) ? -1 : 1;
int is_mfwd = IS_MATE_REVERSE(bam_line) ? -1 : 1;
if ( stats->regions )
{
// Count only on-target bases
- int iref = bam_line->core.pos + 1;
+ hts_pos_t iref = bam_line->core.pos + 1;
for (i=0; i<bam_line->core.n_cigar; i++)
{
int cig = bam_cigar_op(bam_get_cigar(bam_line)[i]);
}
if ( stats->last_pair_tid != bam_line->core.tid) {
- stats->pair_count -= cleanup_overlaps(read_pairs, INT_MAX-1);
+ stats->pair_count -= cleanup_overlaps(read_pairs, INT64_MAX-1);
stats->last_pair_tid = bam_line->core.tid;
stats->last_read_flush = 0;
}
// Coverage distribution graph
round_buffer_flush(stats,bam_line->core.pos);
if ( stats->regions ) {
- uint32_t p = bam_line->core.pos, pnew, pmin, pmax, j;
- pmin = pmax = i = j = 0;
+ hts_pos_t p = bam_line->core.pos, pnew, pmin = 0, pmax = 0;
+ uint32_t j = 0;
+ i = 0;
while ( j < bam_line->core.n_cigar && i < stats->nchunks ) {
int op = bam_cigar_op(bam_get_cigar(bam_line)[j]);
int oplen = bam_cigar_oplen(bam_get_cigar(bam_line)[j]);
case BAM_CMATCH:
case BAM_CEQUAL:
case BAM_CDIFF:
- pmin = MAX(p, stats->chunks[i].from-1);
- pmax = MIN(p+oplen, stats->chunks[i].to);
- if ( pmax >= pmin ) {
+ pmin = MAX(p, stats->chunks[i].beg-1); // 0 based
+ pmax = MIN(p+oplen, stats->chunks[i].end); // 1 based
+ if ( pmax > pmin ) {
if ( stats->info->remove_overlaps )
remove_overlaps(bam_line, read_pairs, stats, pmin, pmax);
else
- round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax-1);
+ round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
}
break;
case BAM_CDEL:
}
pnew = p + (bam_cigar_type(op)&2 ? oplen : 0); // consumes reference
- if ( pnew >= stats->chunks[i].to ) {
+ if ( pnew >= stats->chunks[i].end ) {
// go to the next chunk
i++;
} else {
}
}
} else {
- uint32_t p = bam_line->core.pos, j;
+ hts_pos_t p = bam_line->core.pos;
+ uint32_t j;
for (j = 0; j < bam_line->core.n_cigar; j++) {
int op = bam_cigar_op(bam_get_cigar(bam_line)[j]);
int oplen = bam_cigar_oplen(bam_get_cigar(bam_line)[j]);
if ( stats->info->remove_overlaps )
remove_overlaps(bam_line, read_pairs, stats, p, p+oplen);
else
- round_buffer_insert_read(&(stats->cov_rbuf), p, p+oplen-1);
+ round_buffer_insert_read(&(stats->cov_rbuf), p, p+oplen);
break;
case BAM_CDEL:
break;
}
}
if ( stats->info->remove_overlaps )
- remove_overlaps(bam_line, read_pairs, stats, -1, -1); //remove the line from the hash table
+ remove_overlaps(bam_line, read_pairs, stats, -1LL, -1LL); //remove the line from the hash table
}
}
float n,d;
int k;
- n = p*(N+1)/100;
+ n = (float)p*(N+1)/100;
k = n;
if ( k<=0 )
return gcd[0].depth;
fprintf(to, "# CHK, CRC32 of reads which passed filtering followed by addition (32bit overflow)\n");
fprintf(to, "CHK\t%08x\t%08x\t%08x\n", stats->checksum.names,stats->checksum.reads,stats->checksum.quals);
fprintf(to, "# Summary Numbers. Use `grep ^SN | cut -f 2-` to extract this part.\n");
- fprintf(to, "SN\traw total sequences:\t%ld\n", (long)(stats->nreads_filtered+stats->nreads_1st+stats->nreads_2nd)); // not counting excluded seqs (and none of the below)
+ fprintf(to, "SN\traw total sequences:\t%ld\n", (long)(stats->nreads_filtered+stats->nreads_1st+stats->nreads_2nd+stats->nreads_other)); // not counting excluded seqs (and none of the below)
fprintf(to, "SN\tfiltered sequences:\t%ld\n", (long)stats->nreads_filtered);
- fprintf(to, "SN\tsequences:\t%ld\n", (long)(stats->nreads_1st+stats->nreads_2nd));
+ fprintf(to, "SN\tsequences:\t%ld\n", (long)(stats->nreads_1st+stats->nreads_2nd+stats->nreads_other));
fprintf(to, "SN\tis sorted:\t%d\n", stats->is_sorted ? 1 : 0);
fprintf(to, "SN\t1st fragments:\t%ld\n", (long)stats->nreads_1st);
fprintf(to, "SN\tlast fragments:\t%ld\n", (long)stats->nreads_2nd);
fprintf(to, "SN\tbases duplicated:\t%ld\n", (long)stats->total_len_dup);
fprintf(to, "SN\tmismatches:\t%ld\t# from NM fields\n", (long)stats->nmismatches);
fprintf(to, "SN\terror rate:\t%e\t# mismatches / bases mapped (cigar)\n", stats->nbases_mapped_cigar ? (float)stats->nmismatches/stats->nbases_mapped_cigar : 0);
- float avg_read_length = (stats->nreads_1st+stats->nreads_2nd)?stats->total_len/(stats->nreads_1st+stats->nreads_2nd):0;
+ float avg_read_length = (stats->nreads_1st+stats->nreads_2nd+stats->nreads_other)?stats->total_len/(stats->nreads_1st+stats->nreads_2nd+stats->nreads_other):0;
fprintf(to, "SN\taverage length:\t%.0f\n", avg_read_length);
fprintf(to, "SN\taverage first fragment length:\t%.0f\n", stats->nreads_1st? (float)stats->total_len_1st/stats->nreads_1st:0);
fprintf(to, "SN\taverage last fragment length:\t%.0f\n", stats->nreads_2nd? (float)stats->total_len_2nd/stats->nreads_2nd:0);
fprintf(to, "SN\toutward oriented pairs:\t%ld\n", (long)nisize_outward);
fprintf(to, "SN\tpairs with other orientation:\t%ld\n", (long)nisize_other);
fprintf(to, "SN\tpairs on different chromosomes:\t%ld\n", (long)stats->nreads_anomalous/2);
- fprintf(to, "SN\tpercentage of properly paired reads (%%):\t%.1f\n", (stats->nreads_1st+stats->nreads_2nd)? (float)(100*stats->nreads_properly_paired)/(stats->nreads_1st+stats->nreads_2nd):0);
+ fprintf(to, "SN\tpercentage of properly paired reads (%%):\t%.1f\n", (stats->nreads_1st+stats->nreads_2nd+stats->nreads_other)? (float)(100*stats->nreads_properly_paired)/(stats->nreads_1st+stats->nreads_2nd+stats->nreads_other):0);
if ( stats->target_count ) {
fprintf(to, "SN\tbases inside the target:\t%u\n", stats->target_count);
for (icov=stats->info->cov_threshold+1; icov<stats->ncov; icov++)
100.*(acgtno_count_1st->other + acgtno_count_2nd->other)/acgt_sum);
}
+
+ uint64_t tA=0, tC=0, tG=0, tT=0, tN=0;
fprintf(to, "# ACGT content per cycle for first fragments. Use `grep ^FBC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N and O counts as a percentage of all A/C/G/T bases [%%]\n");
for (ibase=0; ibase<stats->max_len; ibase++)
{
acgtno_count_t *acgtno_count_1st = &(stats->acgtno_cycles_1st[ibase]);
uint64_t acgt_sum_1st = acgtno_count_1st->a + acgtno_count_1st->c + acgtno_count_1st->g + acgtno_count_1st->t;
+ tA += acgtno_count_1st->a;
+ tC += acgtno_count_1st->c;
+ tG += acgtno_count_1st->g;
+ tT += acgtno_count_1st->t;
+ tN += acgtno_count_1st->n;
if ( acgt_sum_1st )
fprintf(to, "FBC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
100.*acgtno_count_1st->other/acgt_sum_1st);
}
+ fprintf(to, "# ACGT raw counters for first fragments. Use `grep ^FTC | cut -f 2-` to extract this part. The columns are: A,C,G,T,N base counters\n");
+ fprintf(to, "FTC\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\n", tA, tC, tG, tT, tN);
+ tA=0, tC=0, tG=0, tT=0, tN=0;
fprintf(to, "# ACGT content per cycle for last fragments. Use `grep ^LBC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N and O counts as a percentage of all A/C/G/T bases [%%]\n");
for (ibase=0; ibase<stats->max_len; ibase++)
{
acgtno_count_t *acgtno_count_2nd = &(stats->acgtno_cycles_2nd[ibase]);
uint64_t acgt_sum_2nd = acgtno_count_2nd->a + acgtno_count_2nd->c + acgtno_count_2nd->g + acgtno_count_2nd->t;
+ tA += acgtno_count_2nd->a;
+ tC += acgtno_count_2nd->c;
+ tG += acgtno_count_2nd->g;
+ tT += acgtno_count_2nd->t;
+ tN += acgtno_count_2nd->n;
if ( acgt_sum_2nd )
fprintf(to, "LBC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
100.*acgtno_count_2nd->other/acgt_sum_2nd);
}
+ fprintf(to, "# ACGT raw counters for last fragments. Use `grep ^LTC | cut -f 2-` to extract this part. The columns are: A,C,G,T,N base counters\n");
+ fprintf(to, "LTC\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\n", tA, tC, tG, tT, tN);
+
+ int tag;
+ for (tag=0; tag<stats->ntags; tag++) {
+ if (stats->tags_barcode[tag].nbases) {
+ fprintf(to, "# ACGT content per cycle for barcodes. Use `grep ^%sC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N counts as a percentage of all A/C/G/T bases [%%]\n",
+ stats->tags_barcode[tag].tag_name);
+ for (ibase=0; ibase<stats->tags_barcode[tag].nbases; ibase++)
+ {
+ if (ibase == stats->tags_barcode[tag].tag_sep)
+ continue;
+
+ acgtno_count_t *acgtno_count = stats->acgtno_barcode + stats->tags_barcode[tag].offset + ibase;
+ uint64_t acgt_sum = acgtno_count->a + acgtno_count->c + acgtno_count->g + acgtno_count->t;
+
+ if ( acgt_sum )
+ fprintf(to, "%sC%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", stats->tags_barcode[tag].tag_name,
+ stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? 1 : 2,
+ stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? ibase+1 : ibase-stats->tags_barcode[tag].tag_sep,
+ 100.*acgtno_count->a/acgt_sum,
+ 100.*acgtno_count->c/acgt_sum,
+ 100.*acgtno_count->g/acgt_sum,
+ 100.*acgtno_count->t/acgt_sum,
+ 100.*acgtno_count->n/acgt_sum);
+ }
+
+ fprintf(to, "# Barcode Qualities. Use `grep ^%sQ | cut -f 2-` to extract this part.\n", stats->tags_barcode[tag].qual_name);
+ fprintf(to, "# Columns correspond to qualities and rows to barcode cycles. First column is the cycle number.\n");
+ for (ibase=0; ibase<stats->tags_barcode[tag].nbases; ibase++)
+ {
+ if (ibase == stats->tags_barcode[tag].tag_sep)
+ continue;
+
+ fprintf(to, "%sQ%d\t%d", stats->tags_barcode[tag].qual_name,
+ stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? 1 : 2,
+ stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? ibase+1 : ibase-stats->tags_barcode[tag].tag_sep);
+ for (iqual=0; iqual<=stats->tags_barcode[tag].max_qual; iqual++)
+ {
+ fprintf(to, "\t%ld", (long)stats->quals_barcode[(stats->tags_barcode[tag].offset + ibase)*stats->nquals+iqual]);
+ }
+ fprintf(to, "\n");
+ }
+ }
+ }
+
fprintf(to, "# Insert sizes. Use `grep ^IS | cut -f 2-` to extract this part. The columns are: insert size, pairs total, inward oriented pairs, outward oriented pairs, other pairs\n");
for (isize=0; isize<ibulk; isize++) {
long in = (long)(stats->isize->inward(stats->isize->data, isize));
}
}
-void init_regions(stats_t *stats, const char *file)
+static void init_regions(stats_t *stats, const char *file)
{
FILE *fp = fopen(file,"r");
if ( !fp ) error("%s: %s\n",file,strerror(errno));
kstring_t line = { 0, 0, NULL };
int warned = 0, r, p, new_p;
- int prev_tid=-1, prev_pos=-1;
+ int prev_tid=-1;
+ hts_pos_t prev_pos=-1LL;
while (line.l = 0, kgetline(&line, (kgets_func *)fgets, fp) >= 0)
{
if ( line.s[0] == '#' ) continue;
if ( tid >= stats->nregions )
{
- stats->regions = realloc(stats->regions,sizeof(regions_t)*(stats->nregions+100));
+ if(!(stats->regions = realloc(stats->regions,sizeof(regions_t)*(tid+REG_INC))))
+ error("Could not allocate memory for region.\n");
+
int j;
- for (j=stats->nregions; j<stats->nregions+100; j++)
+ for (j=stats->nregions; j<tid+REG_INC; j++)
{
stats->regions[j].npos = stats->regions[j].mpos = stats->regions[j].cpos = 0;
stats->regions[j].pos = NULL;
}
- stats->nregions += 100;
+ stats->nregions = tid+REG_INC;
}
int npos = stats->regions[tid].npos;
if ( npos >= stats->regions[tid].mpos )
{
- stats->regions[tid].mpos += 1000;
- stats->regions[tid].pos = realloc(stats->regions[tid].pos,sizeof(pos_t)*stats->regions[tid].mpos);
+ stats->regions[tid].mpos = npos+POS_INC;
+ if (!(stats->regions[tid].pos = realloc(stats->regions[tid].pos, sizeof(hts_pair_pos_t)*stats->regions[tid].mpos)))
+ error("Could not allocate memory for interval.\n");
}
- if ( (sscanf(&line.s[i+1],"%u %u",&stats->regions[tid].pos[npos].from,&stats->regions[tid].pos[npos].to))!=2 ) error("Could not parse the region [%s]\n", &line.s[i+1]);
+ if ( (sscanf(&line.s[i+1],"%"SCNd64" %"SCNd64, &stats->regions[tid].pos[npos].beg, &stats->regions[tid].pos[npos].end))!=2 ) error("Could not parse the region [%s]\n", &line.s[i+1]);
if ( prev_tid==-1 || prev_tid!=tid )
{
prev_tid = tid;
- prev_pos = stats->regions[tid].pos[npos].from;
+ prev_pos = stats->regions[tid].pos[npos].beg;
}
- if ( prev_pos>stats->regions[tid].pos[npos].from )
- error("The positions are not in chromosomal order (%s:%d comes after %d)\n", line.s,stats->regions[tid].pos[npos].from,prev_pos);
+ if ( prev_pos>stats->regions[tid].pos[npos].beg )
+ error("The positions are not in chromosomal order (%s:%"PRIhts_pos" comes after %"PRIhts_pos")\n", line.s, stats->regions[tid].pos[npos].beg, prev_pos);
stats->regions[tid].npos++;
if ( stats->regions[tid].npos > stats->nchunks )
stats->nchunks = stats->regions[tid].npos;
for (r = 0; r < stats->nregions; r++) {
regions_t *reg = &stats->regions[r];
if ( reg->npos > 1 ) {
- qsort(reg->pos, reg->npos, sizeof(pos_t), regions_lt);
+ qsort(reg->pos, reg->npos, sizeof(hts_pair_pos_t), regions_lt);
for (new_p = 0, p = 1; p < reg->npos; p++) {
- if ( reg->pos[new_p].to < reg->pos[p].from )
+ if ( reg->pos[new_p].end < reg->pos[p].beg )
reg->pos[++new_p] = reg->pos[p];
- else if ( reg->pos[new_p].to < reg->pos[p].to )
- reg->pos[new_p].to = reg->pos[p].to;
+ else if ( reg->pos[new_p].end < reg->pos[p].end )
+ reg->pos[new_p].end = reg->pos[p].end;
}
reg->npos = ++new_p;
}
for (p = 0; p < reg->npos; p++)
- stats->target_count += (reg->pos[p].to - reg->pos[p].from + 1);
+ stats->target_count += (reg->pos[p].end - reg->pos[p].beg + 1);
}
- stats->chunks = calloc(stats->nchunks, sizeof(pos_t));
+ if (!(stats->chunks = calloc(stats->nchunks, sizeof(hts_pair_pos_t))))
+ error("Could not allocate memory for chunk.\n");
}
void destroy_regions(stats_t *stats)
// Find a matching interval or skip this read. No splicing of reads is done, no indels or soft clips considered,
// even small overlap is enough to include the read in the stats.
int i = reg->cpos;
- while ( i<reg->npos && reg->pos[i].to<=bam_line->core.pos ) i++;
+ while ( i<reg->npos && reg->pos[i].end<=bam_line->core.pos ) i++;
if ( i>=reg->npos ) { reg->cpos = reg->npos; return 0; }
int64_t endpos = bam_endpos(bam_line);
- if ( endpos < reg->pos[i].from ) return 0;
+ if ( endpos < reg->pos[i].beg ) return 0;
//found a read overlapping a region
reg->cpos = i;
- stats->reg_from = reg->pos[i].from;
- stats->reg_to = reg->pos[i].to;
+ stats->reg_from = reg->pos[i].beg;
+ stats->reg_to = reg->pos[i].end;
//now find all the overlapping chunks
stats->nchunks = 0;
while (i < reg->npos) {
- if (bam_line->core.pos < reg->pos[i].to && endpos >= reg->pos[i].from) {
- stats->chunks[stats->nchunks].from = MAX(bam_line->core.pos+1, reg->pos[i].from);
- stats->chunks[stats->nchunks].to = MIN(endpos, reg->pos[i].to);
+ if (bam_line->core.pos < reg->pos[i].end && endpos >= reg->pos[i].beg) {
+ stats->chunks[stats->nchunks].beg = MAX(bam_line->core.pos+1, reg->pos[i].beg);
+ stats->chunks[stats->nchunks].end = MIN(endpos, reg->pos[i].end);
stats->nchunks++;
}
i++;
int i, j, tid;
stats->nregions = iter->n_reg;
stats->regions = calloc(stats->nregions, sizeof(regions_t));
- stats->chunks = calloc(stats->nchunks, sizeof(pos_t));
+ stats->chunks = calloc(stats->nchunks, sizeof(hts_pair_pos_t));
if ( !stats->regions || !stats->chunks )
return 1;
}
stats->regions[tid].mpos = stats->regions[tid].npos = iter->reg_list[i].count;
- stats->regions[tid].pos = calloc(stats->regions[tid].mpos, sizeof(pos_t));
+ stats->regions[tid].pos = calloc(stats->regions[tid].mpos, sizeof(hts_pair_pos_t));
if ( !stats->regions[tid].pos )
return 1;
for (j = 0; j < stats->regions[tid].npos; j++) {
- stats->regions[tid].pos[j].from = iter->reg_list[i].intervals[j].beg+1;
- stats->regions[tid].pos[j].to = iter->reg_list[i].intervals[j].end;
+ stats->regions[tid].pos[j].beg = iter->reg_list[i].intervals[j].beg+1;
+ stats->regions[tid].pos[j].end = iter->reg_list[i].intervals[j].end;
- stats->target_count += (stats->regions[tid].pos[j].to - stats->regions[tid].pos[j].from + 1);
+ stats->target_count += (stats->regions[tid].pos[j].end - stats->regions[tid].pos[j].beg + 1);
}
}
}
-static void error(const char *format, ...)
+static void HTS_NORETURN error(const char *format, ...)
{
if ( !format )
{
fprintf(samtools_stdout, "Options:\n");
fprintf(samtools_stdout, " -c, --coverage <int>,<int>,<int> Coverage distribution min,max,step [1,1000,1]\n");
fprintf(samtools_stdout, " -d, --remove-dups Exclude from statistics reads marked as duplicates\n");
+ fprintf(samtools_stdout, " -X, --customized-index-file Use a customized index file\n");
fprintf(samtools_stdout, " -f, --required-flag <str|int> Required flag, 0 for unset. See also `samtools flags` [0]\n");
fprintf(samtools_stdout, " -F, --filtering-flag <str|int> Filtering flag, 0 for unset. See also `samtools flags` [0]\n");
fprintf(samtools_stdout, " --GC-depth <float> the size of GC-depth bins (decreasing bin size increases memory requirement) [2e4]\n");
fprintf(samtools_stdout, " -h, --help This help message\n");
fprintf(samtools_stdout, " -i, --insert-size <int> Maximum insert size [8000]\n");
fprintf(samtools_stdout, " -I, --id <string> Include only listed read group or sample name\n");
- fprintf(samtools_stdout, " -l, --read-length <int> Include in the statistics only reads with the given read length []\n");
+ fprintf(samtools_stdout, " -l, --read-length <int> Include in the statistics only reads with the given read length [-1]\n");
fprintf(samtools_stdout, " -m, --most-inserts <float> Report only the main part of inserts [0.99]\n");
fprintf(samtools_stdout, " -P, --split-prefix <str> Path or string prefix for filepaths output by -S (default is input filename)\n");
fprintf(samtools_stdout, " -q, --trim-quality <int> The BWA trimming parameter [0]\n");
fprintf(samtools_stdout, " -t, --target-regions <file> Do stats in these regions only. Tab-delimited file chr,from,to, 1-based, inclusive.\n");
fprintf(samtools_stdout, " -x, --sparse Suppress outputting IS rows where there are no insertions.\n");
fprintf(samtools_stdout, " -p, --remove-overlaps Remove overlaps of paired-end reads from coverage and base count computations.\n");
- fprintf(samtools_stdout, " -g, --cov-threshold Only bases with coverage above this value will be included in the target percentage computation.\n");
- sam_global_opt_help(samtools_stdout, "-.--.@");
+ fprintf(samtools_stdout, " -g, --cov-threshold <int> Only bases with coverage above this value will be included in the target percentage computation [0]\n");
+ sam_global_opt_help(samtools_stdout, "-.--.@-.");
fprintf(samtools_stdout, "\n");
}
else
free(stats->ins_cycles_2nd);
free(stats->del_cycles_1st);
free(stats->del_cycles_2nd);
+ if (stats->acgtno_barcode) free(stats->acgtno_barcode);
+ if (stats->quals_barcode) free(stats->quals_barcode);
+ free(stats->tags_barcode);
destroy_regions(stats);
if ( stats->rg_hash ) khash_str2int_destroy(stats->rg_hash);
free(stats->split_name);
void destroy_split_stats(khash_t(c2stats) *split_hash)
{
+ if (!split_hash)
+ return;
+
int i = 0;
stats_t *curr_stats = NULL;
for(i = kh_begin(split_hash); i != kh_end(split_hash); ++i){
stats_info_t* stats_info_init(int argc, char *argv[])
{
stats_info_t* info = calloc(1, sizeof(stats_info_t));
+ if (!info) {
+ return NULL;
+ }
+
info->nisize = 8000;
info->isize_main_bulk = 0.99; // There are always outliers at the far end
info->gcd_bin_size = 20e3;
stats_t* stats_init()
{
stats_t *stats = calloc(1,sizeof(stats_t));
+ if (!stats)
+ return NULL;
+
stats->ngc = 200;
stats->nquals = 256;
stats->nbases = 300;
stats->rseq_pos = -1;
- stats->tid = stats->gcd_pos = -1;
+ stats->tid = -1;
+ stats->gcd_pos = -1LL;
stats->igcd = 0;
stats->is_sorted = 1;
stats->nindels = stats->nbases;
return stats;
}
+static int init_barcode_tags(stats_t* stats) {
+ stats->ntags = 4;
+ stats->tags_barcode = calloc(stats->ntags, sizeof(barcode_info_t));
+ if (!stats->tags_barcode)
+ return -1;
+ stats->tags_barcode[0] = (barcode_info_t){"BC", "QT", 0, -1, -1, 0};
+ stats->tags_barcode[1] = (barcode_info_t){"CR", "CY", 0, -1, -1, 0};
+ stats->tags_barcode[2] = (barcode_info_t){"OX", "BZ", 0, -1, -1, 0};
+ stats->tags_barcode[3] = (barcode_info_t){"RX", "QX", 0, -1, -1, 0};
+ return 0;
+}
+
static void init_stat_structs(stats_t* stats, stats_info_t* info, const char* group_id, const char* targets)
{
// Give stats_t a pointer to the info struct
stats->ncov = 3 + (info->cov_max-info->cov_min) / info->cov_step;
info->cov_max = info->cov_min + ((info->cov_max-info->cov_min)/info->cov_step +1)*info->cov_step - 1;
stats->cov = calloc(sizeof(uint64_t),stats->ncov);
+ if (!stats->cov) goto nomem;
stats->cov_rbuf.size = stats->nbases*5;
stats->cov_rbuf.buffer = calloc(sizeof(int32_t),stats->cov_rbuf.size);
-
+ if (!stats->cov_rbuf.buffer) goto nomem;
if ( group_id ) init_group_id(stats, group_id);
// .. arrays
stats->quals_1st = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
+ if (!stats->quals_1st) goto nomem;
stats->quals_2nd = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
+ if (!stats->quals_2nd) goto nomem;
stats->gc_1st = calloc(stats->ngc,sizeof(uint64_t));
+ if (!stats->gc_1st) goto nomem;
stats->gc_2nd = calloc(stats->ngc,sizeof(uint64_t));
+ if (!stats->gc_2nd) goto nomem;
stats->isize = init_isize_t(info->nisize ?info->nisize+1 :0);
+ if (!stats->isize) goto nomem;
stats->gcd = calloc(stats->ngcd,sizeof(gc_depth_t));
- stats->mpc_buf = info->fai ? calloc(stats->nquals*stats->nbases,sizeof(uint64_t)) : NULL;
+ if (!stats->gcd) goto nomem;
+ if (info->fai) {
+ stats->mpc_buf = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
+ if (!stats->mpc_buf) goto nomem;
+ } else {
+ stats->mpc_buf = NULL;
+ }
stats->acgtno_cycles_1st = calloc(stats->nbases,sizeof(acgtno_count_t));
+ if (!stats->acgtno_cycles_1st) goto nomem;
stats->acgtno_cycles_2nd = calloc(stats->nbases,sizeof(acgtno_count_t));
+ if (!stats->acgtno_cycles_2nd) goto nomem;
stats->read_lengths = calloc(stats->nbases,sizeof(uint64_t));
+ if (!stats->read_lengths) goto nomem;
stats->read_lengths_1st = calloc(stats->nbases,sizeof(uint64_t));
+ if (!stats->read_lengths_1st) goto nomem;
stats->read_lengths_2nd = calloc(stats->nbases,sizeof(uint64_t));
+ if (!stats->read_lengths_2nd) goto nomem;
stats->insertions = calloc(stats->nbases,sizeof(uint64_t));
+ if (!stats->insertions) goto nomem;
stats->deletions = calloc(stats->nbases,sizeof(uint64_t));
+ if (!stats->deletions) goto nomem;
stats->ins_cycles_1st = calloc(stats->nbases+1,sizeof(uint64_t));
+ if (!stats->ins_cycles_1st) goto nomem;
stats->ins_cycles_2nd = calloc(stats->nbases+1,sizeof(uint64_t));
+ if (!stats->ins_cycles_2nd) goto nomem;
stats->del_cycles_1st = calloc(stats->nbases+1,sizeof(uint64_t));
+ if (!stats->del_cycles_1st) goto nomem;
stats->del_cycles_2nd = calloc(stats->nbases+1,sizeof(uint64_t));
+ if (!stats->del_cycles_2nd) goto nomem;
+ if (init_barcode_tags(stats) < 0)
+ goto nomem;
realloc_rseq_buffer(stats);
if ( targets )
init_regions(stats, targets);
+ return;
+ nomem:
+ error("Out of memory");
}
static stats_t* get_curr_split_stats(bam1_t* bam_line, khash_t(c2stats)* split_hash, stats_info_t* info, char* targets)
khiter_t k = kh_get(c2stats, split_hash, split_name);
if(k == kh_end(split_hash)){
curr_stats = stats_init(); // mallocs new instance
+ if (!curr_stats) {
+ error("Couldn't allocate split stats");
+ }
init_stat_structs(curr_stats, info, NULL, targets);
curr_stats->split_name = split_name;
{
char *targets = NULL;
char *bam_fname = NULL;
+ char *bam_idx_fname = NULL;
char *group_id = NULL;
- int sparse = 0;
+ int sparse = 0, has_index_file = 0, ret = 1;
sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
stats_info_t *info = stats_info_init(argc, argv);
+ if (!info) {
+ fprintf(samtools_stderr, "Could not allocate memory for info.\n");
+ return 1;
+ }
static const struct option loptions[] =
{
{"help", no_argument, NULL, 'h'},
{"remove-dups", no_argument, NULL, 'd'},
{"sam", no_argument, NULL, 's'},
+ {"customized-index-file", required_argument, NULL, 'X'},
{"ref-seq", required_argument, NULL, 'r'},
{"coverage", required_argument, NULL, 'c'},
{"read-length", required_argument, NULL, 'l'},
};
int opt;
- while ( (opt=getopt_long(argc,argv,"?hdsxpr:c:l:i:t:m:q:f:F:g:I:1:S:P:@:",loptions,NULL))>0 )
+ while ( (opt=getopt_long(argc,argv,"?hdsXxpr:c:l:i:t:m:q:f:F:g:I:S:P:@:",loptions,NULL))>0 )
{
switch (opt)
{
case 'f': info->flag_require = bam_str2flag(optarg); break;
case 'F': info->flag_filter |= bam_str2flag(optarg); break;
case 'd': info->flag_filter |= BAM_FDUP; break;
+ case 'X': has_index_file = 1; break;
case 's': break;
case 'r': info->fai = fai_load(optarg);
if (info->fai==NULL)
break;
case '?':
case 'h': error(NULL);
+ /* no break */
default:
if (parse_sam_global_opt(opt, optarg, loptions, &ga) != 0)
error("Unknown argument: %s\n", optarg);
break;
}
}
- if ( optind<argc )
- bam_fname = argv[optind++];
+ bam_fname = argv[optind++];
if ( !bam_fname )
{
if ( isatty(STDIN_FILENO) )
free(info);
return 1;
}
+
+ if (has_index_file && !(bam_idx_fname = argv[optind++])) {
+ fprintf(samtools_stderr, "No index file provided\n");
+ free(info);
+ return 1;
+ }
+
if (ga.nthreads > 0)
hts_set_threads(info->sam, ga.nthreads);
stats_t *all_stats = stats_init();
+ if (!all_stats) {
+ fprintf(samtools_stderr, "Could not allocate memory for stats.\n");
+ cleanup_stats_info(info);
+ return 1;
+ }
stats_t *curr_stats = NULL;
init_stat_structs(all_stats, info, group_id, targets);
// Init
// .. hash
khash_t(c2stats)* split_hash = kh_init(c2stats);
+ if (!split_hash) goto cleanup_all_stats;
khash_t(qn2pair)* read_pairs = kh_init(qn2pair);
+ if (!read_pairs) goto cleanup_split_hash;
// Collect statistics
bam1_t *bam_line = bam_init1();
- if ( optind<argc )
- {
- int filter = 1;
- // Prepare the region hash table for the multi-region iterator
- void *region_hash = bed_hash_regions(NULL, argv, optind, argc, &filter);
- if (region_hash) {
-
- // Collect stats in selected regions only
- hts_idx_t *bam_idx = sam_index_load(info->sam,bam_fname);
- if (bam_idx) {
-
- int regcount = 0;
- hts_reglist_t *reglist = bed_reglist(region_hash, ALL, ®count);
- if (reglist) {
-
- hts_itr_multi_t *iter = sam_itr_regions(bam_idx, info->sam_header, reglist, regcount);
- if (iter) {
-
- if (!targets) {
- all_stats->nchunks = argc-optind;
- if ( replicate_regions(all_stats, iter) )
- fprintf(samtools_stderr, "Replications of the regions failed.");
- }
+ if (!bam_line) goto cleanup_read_pairs;
- if ( all_stats->nregions && all_stats->regions ) {
- while (sam_itr_multi_next(info->sam, iter, bam_line) >= 0) {
- if (info->split_tag) {
- curr_stats = get_curr_split_stats(bam_line, split_hash, info, targets);
- collect_stats(bam_line, curr_stats, read_pairs);
- }
- collect_stats(bam_line, all_stats, read_pairs);
- }
+ if (optind < argc) {
+ // Region:interval arguments in the command line
+ hts_idx_t *bam_idx = NULL;
+ if (has_index_file) {
+ bam_idx = sam_index_load2(info->sam, bam_fname, bam_idx_fname);
+ } else {
+ // If an index filename has not been specified, look alongside the alignment file
+ bam_idx = sam_index_load(info->sam, bam_fname);
+ }
+
+ if (bam_idx) {
+ hts_itr_multi_t *iter = sam_itr_regarray(bam_idx, info->sam_header, &argv[optind], argc - optind);
+ if (iter) {
+ if (!targets) {
+ all_stats->nchunks = argc-optind;
+ if (replicate_regions(all_stats, iter))
+ fprintf(samtools_stderr, "Replications of the regions failed\n");
+ }
+
+ if ( all_stats->nregions && all_stats->regions ) {
+ while ((ret = sam_itr_next(info->sam, iter, bam_line)) >= 0) {
+ if (info->split_tag) {
+ curr_stats = get_curr_split_stats(bam_line, split_hash, info, targets);
+ collect_stats(bam_line, curr_stats, read_pairs);
}
+ collect_stats(bam_line, all_stats, read_pairs);
+ }
+ if (ret < -1) {
+ fprintf(samtools_stderr, "Failure while running the iterator\n");
hts_itr_multi_destroy(iter);
- } else {
- fprintf(samtools_stderr, "Creation of the region iterator failed.");
- hts_reglist_free(reglist, regcount);
+ hts_idx_destroy(bam_idx);
+ goto cleanup;
}
- } else {
- fprintf(samtools_stderr, "Creation of the region list failed.");
}
-
- hts_idx_destroy(bam_idx);
+ hts_itr_multi_destroy(iter);
} else {
- fprintf(samtools_stderr, "Random alignment retrieval only works for indexed BAM files.\n");
+ fprintf(samtools_stderr, "Multi-region iterator could not be created\n");
+ hts_idx_destroy(bam_idx);
+ goto cleanup;
}
-
- bed_destroy(region_hash);
+ hts_idx_destroy(bam_idx);
} else {
- fprintf(samtools_stderr, "Creation of the region hash table failed.\n");
+ if (has_index_file)
+ fprintf(samtools_stderr, "Invalid index file '%s'\n", bam_idx_fname);
+ fprintf(samtools_stderr, "Random alignment retrieval only works for indexed files\n");
+ goto cleanup;
}
- }
- else
- {
+ } else {
if ( info->cov_threshold > 0 && !targets ) {
- fprintf(samtools_stderr, "Coverage percentage calcuation requires a list of target regions\n");
+ fprintf(samtools_stderr, "Coverage percentage calculation requires a list of target regions\n");
goto cleanup;
}
// Stream through the entire BAM ignoring off-target regions if -t is given
- int ret;
while ((ret = sam_read1(info->sam, info->sam_header, bam_line)) >= 0) {
if (info->split_tag) {
curr_stats = get_curr_split_stats(bam_line, split_hash, info, targets);
if (ret < -1) {
fprintf(samtools_stderr, "Failure while decoding file\n");
- return 1;
+ goto cleanup;
}
}
if (info->split_tag)
output_split_stats(split_hash, bam_fname, sparse);
+ ret = 0;
cleanup:
bam_destroy1(bam_line);
- bam_hdr_destroy(info->sam_header);
+ sam_hdr_destroy(info->sam_header);
sam_global_args_free(&ga);
+cleanup_read_pairs:
+ cleanup_overlaps(read_pairs, INT64_MAX);
+cleanup_split_hash:
+ destroy_split_stats(split_hash);
+cleanup_all_stats:
cleanup_stats(all_stats);
cleanup_stats_info(info);
- destroy_split_stats(split_hash);
- cleanup_overlaps(read_pairs, INT_MAX);
- return 0;
+ return ret;
}
/* stats_isize.c -- generalised insert size calculation for samtools stats.
- Copyright (C) 2014 Genome Research Ltd.
+ Copyright (C) 2014, 2018 Genome Research Ltd.
Author: Nicholas Clarke <nc6@sanger.ac.uk>
if (bound <= 0) {
// Use sparse data structure.
isize_sparse_data_t *data = (isize_sparse_data_t *) malloc(sizeof(isize_sparse_data_t));
+ if (!data)
+ return NULL;
// Initialise
data->max = 0;
data->array = kh_init(m32);
+ if (!data->array) {
+ free(data);
+ return NULL;
+ }
isize_t *isize = (isize_t *)malloc(sizeof(isize_t));
+ if (!isize) {
+ kh_destroy(m32, data->array);
+ free(data);
+ return NULL;
+ }
isize->data.sparse = data;
isize->nitems = & sparse_nitems;
uint64_t* out = calloc(bound,sizeof(uint64_t));
uint64_t* other = calloc(bound,sizeof(uint64_t));
isize_dense_data_t *rec = (isize_dense_data_t *)malloc(sizeof(isize_dense_data_t));
+ isize_t *isize = (isize_t *)malloc(sizeof(isize_t));
+ if (!in || !out || !other || !rec || !isize) {
+ free(in);
+ free(out);
+ free(other);
+ free(rec);
+ free(isize);
+ return NULL;
+ }
rec->isize_inward = in;
rec->isize_outward = out;
rec->isize_other = other;
rec->total=bound;
- isize_t *isize = (isize_t *)malloc(sizeof(isize_t));
-
isize->data.dense = rec;
isize->nitems = & dense_nitems;
/* stats_isize.c -- generalised insert size calculation for samtools stats.
- Copyright (C) 2014 Genome Research Ltd.
+ Copyright (C) 2014, 2018 Genome Research Ltd.
Author: Nicholas Clarke <nc6@sanger.ac.uk>
if (bound <= 0) {
// Use sparse data structure.
isize_sparse_data_t *data = (isize_sparse_data_t *) malloc(sizeof(isize_sparse_data_t));
+ if (!data)
+ return NULL;
// Initialise
data->max = 0;
data->array = kh_init(m32);
+ if (!data->array) {
+ free(data);
+ return NULL;
+ }
isize_t *isize = (isize_t *)malloc(sizeof(isize_t));
+ if (!isize) {
+ kh_destroy(m32, data->array);
+ free(data);
+ return NULL;
+ }
isize->data.sparse = data;
isize->nitems = & sparse_nitems;
uint64_t* out = calloc(bound,sizeof(uint64_t));
uint64_t* other = calloc(bound,sizeof(uint64_t));
isize_dense_data_t *rec = (isize_dense_data_t *)malloc(sizeof(isize_dense_data_t));
+ isize_t *isize = (isize_t *)malloc(sizeof(isize_t));
+ if (!in || !out || !other || !rec || !isize) {
+ free(in);
+ free(out);
+ free(other);
+ free(rec);
+ free(isize);
+ return NULL;
+ }
rec->isize_inward = in;
rec->isize_outward = out;
rec->isize_other = other;
rec->total=bound;
- isize_t *isize = (isize_t *)malloc(sizeof(isize_t));
-
isize->data.dense = rec;
isize->nitems = & dense_nitems;
+++ /dev/null
-/* test/merge/test_bam_translate.c -- header merging test harness.
-
- Copyright (C) 2013, 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_sort.c"
-#include "../test.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-
-void dump_read(bam1_t* b) {
- printf("->core.tid:(%d)\n", b->core.tid);
- printf("->core.pos:(%d)\n", b->core.pos);
- printf("->core.bin:(%d)\n", b->core.bin);
- printf("->core.qual:(%d)\n", b->core.qual);
- printf("->core.l_qname:(%d)\n", b->core.l_qname);
- printf("->core.flag:(%d)\n", b->core.flag);
- printf("->core.n_cigar:(%d)\n", b->core.n_cigar);
- printf("->core.l_qseq:(%d)\n", b->core.l_qseq);
- printf("->core.mtid:(%d)\n", b->core.mtid);
- printf("->core.mpos:(%d)\n", b->core.mpos);
- printf("->core.isize:(%d)\n", b->core.isize);
- if (b->data) {
- printf("->data:");
- int i;
- for (i = 0; i < b->l_data; ++i) {
- printf("%x ", b->data[i]);
- }
- printf("\n");
- }
- if (b->core.l_qname) {
- printf("qname: %s\n",bam_get_qname(b));
- }
- if (b->core.l_qseq) {
- printf("qseq:");
- int i;
- for (i = 0; i < b->core.l_qseq; ++i) {
- printf("%c",seq_nt16_str[seq_nt16_table[bam_seqi(bam_get_seq(b),i)]]);
- }
- printf("\n");
- printf("qual:");
- for (i = 0; i < b->core.l_qseq; ++i) {
- printf("%c",bam_get_qual(b)[i]);
- }
- printf("\n");
-
- }
-
- if (bam_get_l_aux(b)) {
- int i = 0;
- uint8_t* aux = bam_get_aux(b);
-
- while (i < bam_get_l_aux(b)) {
- printf("%.2s:%c:",aux+i,*(aux+i+2));
- i += 2;
- switch (*(aux+i)) {
- case 'Z':
- while (*(aux+1+i) != '\0') { putc(*(aux+1+i), stdout); ++i; }
- break;
- }
- putc('\n',stdout);
- ++i;++i;
- }
- }
- printf("\n");
-}
-
-void trans_tbl_test_init(trans_tbl_t* tbl, int32_t n_targets)
-{
- tbl->tid_trans = (int*)calloc(n_targets, sizeof(int32_t));
- tbl->rg_trans = kh_init(c2c);
- tbl->pg_trans = kh_init(c2c);
-}
-
-void setup_test_1(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
-
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 0;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-void setup_test_2(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
- int in_there = 0;
- khiter_t iter = kh_put(c2c, tbl->rg_trans, strdup("hello"), &in_there);
- kh_value(tbl->rg_trans, iter) = strdup("goodbye");
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 9;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "RGZhello\0" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-void setup_test_3(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
- int in_there = 0;
- khiter_t iter = kh_put(c2c, tbl->pg_trans, strdup("hello"), &in_there);
- kh_value(tbl->pg_trans,iter) = strdup("goodbye");
-
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 9;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "PGZhello\0" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-void setup_test_4(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 12;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "RGZrg4hello\0" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-void setup_test_5(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
-
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 12;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "PGZpg5hello\0" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-void setup_test_6(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
- int in_there = 0;
- khiter_t iter_rg = kh_put(c2c, tbl->rg_trans, strdup("hello"), &in_there);
- kh_value(tbl->rg_trans, iter_rg) = strdup("goodbye");
- khiter_t iter_pg = kh_put(c2c, tbl->pg_trans, strdup("quail"), &in_there);
- kh_value(tbl->pg_trans, iter_pg) = strdup("bird");
-
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 18;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "RGZhello\0PGZquail\0" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-
-int main(int argc, char**argv)
-{
- // test state
- const int NUM_TESTS = 6;
- int verbose = 0;
- int success = 0;
- int failure = 0;
-
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- break;
- }
- }
-
- bam1_t* b;
-
- // Setup stderr redirect
- kstring_t res = { 0, 0, NULL };
- FILE* orig_stderr = fdopen(dup(STDERR_FILENO), "a"); // Save stderr
- char* tempfname = (optind < argc)? argv[optind] : "test_bam_translate.tmp";
- FILE* check = NULL;
-
- // setup
- if (verbose) printf("BEGIN test 1\n"); // TID test
- trans_tbl_t tbl1;
- setup_test_1(&b,&tbl1);
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
- if (verbose) printf("RUN test 1\n");
-
- // test
- xfreopen(tempfname, "w", stderr); // Redirect stderr to pipe
- bam_translate(b, &tbl1);
- fclose(stderr);
-
- if (verbose) printf("END RUN test 1\n");
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
-
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) < 0 &&
- (feof(check) || res.l == 0) ) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 1\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl1);
- if (verbose) printf("END test 1\n");
-
- // setup
- if (verbose) printf("BEGIN test 2\n"); // RG exists and translate test
- trans_tbl_t tbl2;
- setup_test_2(&b,&tbl2);
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
- if (verbose) printf("RUN test 2\n");
-
- // test
- xfreopen(tempfname, "w", stderr); // Redirect stderr to pipe
- bam_translate(b, &tbl2);
- fclose(stderr);
-
- if (verbose) printf("END RUN test 2\n");
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
-
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) < 0 &&
- (feof(check) || res.l == 0) ) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 2\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl2);
- if (verbose) printf("END test 2\n");
-
- if (verbose) printf("BEGIN test 3\n"); // PG exists and translate test
- // setup
- trans_tbl_t tbl3;
- setup_test_3(&b,&tbl3);
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
- if (verbose) printf("RUN test 3\n");
-
- // test
- xfreopen(tempfname, "w", stderr); // Redirect stderr to pipe
- bam_translate(b, &tbl3);
- fclose(stderr);
-
- if (verbose) printf("END RUN test 3\n");
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
-
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) < 0 &&
- (feof(check) || res.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 3\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl3);
- if (verbose) printf("END test 3\n");
-
- if (verbose) printf("BEGIN test 4\n"); // RG test non-existent
- // setup
- trans_tbl_t tbl4;
- setup_test_4(&b,&tbl4);
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
- if (verbose) printf("RUN test 4\n");
-
- // test
- xfreopen(tempfname, "w", stderr); // Redirect stderr to pipe
- bam_translate(b, &tbl4);
- fclose(stderr);
-
- if (verbose) printf("END RUN test 4\n");
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) >= 0 &&
- strcmp("[bam_translate] RG tag \"rg4hello\" on read \"123456789\" encountered with no corresponding entry in header, tag lost. Unknown tags are only reported once per input file for each tag ID.",res.s) == 0) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 4\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl4);
- if (verbose) printf("END test 4\n");
-
- if (verbose) printf("BEGIN test 5\n"); // PG test non-existent
- // setup
- trans_tbl_t tbl5;
- setup_test_5(&b,&tbl5);
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- printf("RUN test 5\n");
- }
- // test
- xfreopen(tempfname, "w", stderr); // Redirect stderr to pipe
- bam_translate(b, &tbl5);
- fclose(stderr);
-
- if (verbose) printf("END RUN test 5\n");
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
-
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) >= 0 &&
- strcmp("[bam_translate] PG tag \"pg5hello\" on read \"123456789\" encountered with no corresponding entry in header, tag lost. Unknown tags are only reported once per input file for each tag ID.",res.s) == 0) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 5\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl5);
- if (verbose) printf("END test 5\n");
-
- if (verbose) printf("BEGIN test 6\n"); // RG and PG exists and translate test
- // setup
- trans_tbl_t tbl6;
- setup_test_6(&b,&tbl6);
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
- if (verbose) printf("RUN test 6\n");
-
- // test
- xfreopen(tempfname, "w", stderr); // Redirect stderr to pipe
- bam_translate(b, &tbl6);
- fclose(stderr);
-
- if (verbose) printf("END RUN test 6\n");
- if (verbose > 1) {
- printf("b\n");
- dump_read(b);
- }
-
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) < 0 &&
- (feof(check) || res.l == 0) ) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 6\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl6);
- if (verbose) printf("END test 6\n");
-
- // Cleanup
- free(res.s);
- remove(tempfname);
- if (failure > 0)
- fprintf(orig_stderr, "%d failures %d successes\n", failure, success);
- fclose(orig_stderr);
-
- return (success == NUM_TESTS)? EXIT_SUCCESS : EXIT_FAILURE;
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* test/merge/test_bam_translate.c -- header merging test harness.
-
- Copyright (C) 2013, 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_sort.c"
-#include "../test.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-
-void dump_read(bam1_t* b) {
- fprintf(samtools_stdout, "->core.tid:(%d)\n", b->core.tid);
- fprintf(samtools_stdout, "->core.pos:(%d)\n", b->core.pos);
- fprintf(samtools_stdout, "->core.bin:(%d)\n", b->core.bin);
- fprintf(samtools_stdout, "->core.qual:(%d)\n", b->core.qual);
- fprintf(samtools_stdout, "->core.l_qname:(%d)\n", b->core.l_qname);
- fprintf(samtools_stdout, "->core.flag:(%d)\n", b->core.flag);
- fprintf(samtools_stdout, "->core.n_cigar:(%d)\n", b->core.n_cigar);
- fprintf(samtools_stdout, "->core.l_qseq:(%d)\n", b->core.l_qseq);
- fprintf(samtools_stdout, "->core.mtid:(%d)\n", b->core.mtid);
- fprintf(samtools_stdout, "->core.mpos:(%d)\n", b->core.mpos);
- fprintf(samtools_stdout, "->core.isize:(%d)\n", b->core.isize);
- if (b->data) {
- fprintf(samtools_stdout, "->data:");
- int i;
- for (i = 0; i < b->l_data; ++i) {
- fprintf(samtools_stdout, "%x ", b->data[i]);
- }
- fprintf(samtools_stdout, "\n");
- }
- if (b->core.l_qname) {
- fprintf(samtools_stdout, "qname: %s\n",bam_get_qname(b));
- }
- if (b->core.l_qseq) {
- fprintf(samtools_stdout, "qseq:");
- int i;
- for (i = 0; i < b->core.l_qseq; ++i) {
- fprintf(samtools_stdout, "%c",seq_nt16_str[seq_nt16_table[bam_seqi(bam_get_seq(b),i)]]);
- }
- fprintf(samtools_stdout, "\n");
- fprintf(samtools_stdout, "qual:");
- for (i = 0; i < b->core.l_qseq; ++i) {
- fprintf(samtools_stdout, "%c",bam_get_qual(b)[i]);
- }
- fprintf(samtools_stdout, "\n");
-
- }
-
- if (bam_get_l_aux(b)) {
- int i = 0;
- uint8_t* aux = bam_get_aux(b);
-
- while (i < bam_get_l_aux(b)) {
- fprintf(samtools_stdout, "%.2s:%c:",aux+i,*(aux+i+2));
- i += 2;
- switch (*(aux+i)) {
- case 'Z':
- while (*(aux+1+i) != '\0') { putc(*(aux+1+i), samtools_stdout); ++i; }
- break;
- }
- putc('\n',samtools_stdout);
- ++i;++i;
- }
- }
- fprintf(samtools_stdout, "\n");
-}
-
-void trans_tbl_test_init(trans_tbl_t* tbl, int32_t n_targets)
-{
- tbl->tid_trans = (int*)calloc(n_targets, sizeof(int32_t));
- tbl->rg_trans = kh_init(c2c);
- tbl->pg_trans = kh_init(c2c);
-}
-
-void setup_test_1(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
-
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 0;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-void setup_test_2(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
- int in_there = 0;
- khiter_t iter = kh_put(c2c, tbl->rg_trans, strdup("hello"), &in_there);
- kh_value(tbl->rg_trans, iter) = strdup("goodbye");
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 9;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "RGZhello\0" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-void setup_test_3(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
- int in_there = 0;
- khiter_t iter = kh_put(c2c, tbl->pg_trans, strdup("hello"), &in_there);
- kh_value(tbl->pg_trans,iter) = strdup("goodbye");
-
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 9;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "PGZhello\0" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-void setup_test_4(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 12;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "RGZrg4hello\0" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-void setup_test_5(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
-
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 12;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "PGZpg5hello\0" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-void setup_test_6(bam1_t** b_in, trans_tbl_t* tbl) {
- bam1_t* b;
-
- b = bam_init1();
- trans_tbl_test_init(tbl, 4);
-
- tbl->tid_trans[0] = 5;
- tbl->tid_trans[1] = 6;
- tbl->tid_trans[2] = 7;
- tbl->tid_trans[3] = 8;
- int in_there = 0;
- khiter_t iter_rg = kh_put(c2c, tbl->rg_trans, strdup("hello"), &in_there);
- kh_value(tbl->rg_trans, iter_rg) = strdup("goodbye");
- khiter_t iter_pg = kh_put(c2c, tbl->pg_trans, strdup("quail"), &in_there);
- kh_value(tbl->pg_trans, iter_pg) = strdup("bird");
-
-
- b->core.tid = 0;
- b->core.pos = 1334;
- b->core.bin = 0;
- b->core.qual = 10;
- b->core.l_qname = 10;
- b->core.flag = 0;
- b->core.n_cigar = 1;
- b->core.l_qseq = 10;
- b->core.mtid = -1;
- b->core.mpos = 0;
- b->core.isize = -1;
- size_t data_len = 10 + 4 + 5 + 10 + 18;
- b->data = (uint8_t*)malloc(data_len);
- memcpy(b->data,
- "123456789\0" // q_name
- "\x00\x00\x00\xA0" // cigar
- "\x00\x00\x00\x00\x00" // qseq
- "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" // qual
- "RGZhello\0PGZquail\0" // aux
- , data_len
- );
- b->m_data = b->l_data = data_len;
-
- *b_in = b;
-}
-
-
-int samtools_test_bam_translate_main(int argc, char**argv)
-{
- // test state
- const int NUM_TESTS = 6;
- int verbose = 0;
- int success = 0;
- int failure = 0;
-
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- break;
- }
- }
-
- bam1_t* b;
-
- // Setup samtools_stderr redirect
- kstring_t res = { 0, 0, NULL };
- FILE* orig_samtools_stderr = fdopen(dup(STDERR_FILENO), "a"); // Save samtools_stderr
- char* tempfname = (optind < argc)? argv[optind] : "test_bam_translate.tmp";
- FILE* check = NULL;
-
- // setup
- if (verbose) fprintf(samtools_stdout, "BEGIN test 1\n"); // TID test
- trans_tbl_t tbl1;
- setup_test_1(&b,&tbl1);
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 1\n");
-
- // test
- xfreopen(tempfname, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- bam_translate(b, &tbl1);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(samtools_stdout, "END RUN test 1\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
-
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) < 0 &&
- (feof(check) || res.l == 0) ) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 1\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl1);
- if (verbose) fprintf(samtools_stdout, "END test 1\n");
-
- // setup
- if (verbose) fprintf(samtools_stdout, "BEGIN test 2\n"); // RG exists and translate test
- trans_tbl_t tbl2;
- setup_test_2(&b,&tbl2);
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 2\n");
-
- // test
- xfreopen(tempfname, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- bam_translate(b, &tbl2);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(samtools_stdout, "END RUN test 2\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
-
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) < 0 &&
- (feof(check) || res.l == 0) ) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 2\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl2);
- if (verbose) fprintf(samtools_stdout, "END test 2\n");
-
- if (verbose) fprintf(samtools_stdout, "BEGIN test 3\n"); // PG exists and translate test
- // setup
- trans_tbl_t tbl3;
- setup_test_3(&b,&tbl3);
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 3\n");
-
- // test
- xfreopen(tempfname, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- bam_translate(b, &tbl3);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(samtools_stdout, "END RUN test 3\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
-
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) < 0 &&
- (feof(check) || res.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 3\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl3);
- if (verbose) fprintf(samtools_stdout, "END test 3\n");
-
- if (verbose) fprintf(samtools_stdout, "BEGIN test 4\n"); // RG test non-existent
- // setup
- trans_tbl_t tbl4;
- setup_test_4(&b,&tbl4);
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 4\n");
-
- // test
- xfreopen(tempfname, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- bam_translate(b, &tbl4);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(samtools_stdout, "END RUN test 4\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) >= 0 &&
- strcmp("[bam_translate] RG tag \"rg4hello\" on read \"123456789\" encountered with no corresponding entry in header, tag lost. Unknown tags are only reported once per input file for each tag ID.",res.s) == 0) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 4\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl4);
- if (verbose) fprintf(samtools_stdout, "END test 4\n");
-
- if (verbose) fprintf(samtools_stdout, "BEGIN test 5\n"); // PG test non-existent
- // setup
- trans_tbl_t tbl5;
- setup_test_5(&b,&tbl5);
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- fprintf(samtools_stdout, "RUN test 5\n");
- }
- // test
- xfreopen(tempfname, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- bam_translate(b, &tbl5);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(samtools_stdout, "END RUN test 5\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
-
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) >= 0 &&
- strcmp("[bam_translate] PG tag \"pg5hello\" on read \"123456789\" encountered with no corresponding entry in header, tag lost. Unknown tags are only reported once per input file for each tag ID.",res.s) == 0) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 5\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl5);
- if (verbose) fprintf(samtools_stdout, "END test 5\n");
-
- if (verbose) fprintf(samtools_stdout, "BEGIN test 6\n"); // RG and PG exists and translate test
- // setup
- trans_tbl_t tbl6;
- setup_test_6(&b,&tbl6);
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 6\n");
-
- // test
- xfreopen(tempfname, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- bam_translate(b, &tbl6);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(samtools_stdout, "END RUN test 6\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_read(b);
- }
-
- // check result
- check = fopen(tempfname, "r");
- res.l = 0;
- if (kgetline(&res, (kgets_func *)fgets, check) < 0 &&
- (feof(check) || res.l == 0) ) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 6\n");
- }
- fclose(check);
-
- // teardown
- bam_destroy1(b);
- trans_tbl_destroy(&tbl6);
- if (verbose) fprintf(samtools_stdout, "END test 6\n");
-
- // Cleanup
- free(res.s);
- remove(tempfname);
- if (failure > 0)
- fprintf(orig_samtools_stderr, "%d failures %d successes\n", failure, success);
- fclose(orig_samtools_stderr);
-
- return (success == NUM_TESTS)? EXIT_SUCCESS : EXIT_FAILURE;
-}
+++ /dev/null
-/* test/merge/test_rtrans_build.c -- header translation test harness.
-
- Copyright (C) 2013, 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_sort.c"
-
-void dump_rtrans(int* rtrans, int n, int n_targets) {
- printf("->n_targets:(%d)\n", n_targets);
- int i, j;
- for (i = 0; i < n; ++i) {
- fprintf(stderr, "%d",rtrans[i*n_targets+0]);
- for (j = 1; j < n_targets; ++j)
- fprintf(stderr, "\t%d",rtrans[i*n_targets+j]);
- fprintf(stderr, "\n");
- }
-}
-
-void setup_test_1(trans_tbl_t* tbl) {
- tbl[0].n_targets = 2;
- tbl[0].tid_trans = calloc(sizeof(int), 2);
- tbl[0].tid_trans[0] = 0;
- tbl[0].tid_trans[1] = 1;
- tbl[0].rg_trans = kh_init(c2c);
- tbl[0].pg_trans = kh_init(c2c);
-
- tbl[1].n_targets = 2;
- tbl[1].tid_trans = calloc(sizeof(int), 2);
- tbl[1].tid_trans[0] = 1;
- tbl[1].tid_trans[1] = 2;
- tbl[1].rg_trans = kh_init(c2c);
- tbl[1].pg_trans = kh_init(c2c);
-}
-
-bool check_test_1(trans_tbl_t* tbl, int* rtrans) {
- // Check input is unchanged
-
- // Check output
-
- return true;
-}
-
-
-int main(int argc, char**argv)
-{
- const int NUM_TESTS = 1;
- int verbose = 0;
- int success = 0;
- int failure = 0;
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- break;
- }
- }
- const long GIMMICK_SEED = 0x1234330e;
- srand48(GIMMICK_SEED);
-
- if (verbose) printf("BEGIN test 1\n");
- // setup
- trans_tbl_t tbl_1[2];
- int n_targets_1 = 3;
- int n_1 = 2;
- int* rtrans_1 = NULL;
- setup_test_1(&tbl_1[0]);
- // test
- if (verbose > 1) {
- // dump_trans_tid
- }
- if (verbose) printf("RUN test 1\n");
- rtrans_1 = rtrans_build(n_1, n_targets_1, &tbl_1[0]);
- if (verbose) printf("END RUN test 1\n");
- if (verbose > 1) {
- printf("rtrans\n");
- dump_rtrans(rtrans_1, n_1, n_targets_1);
- }
- if (check_test_1(&tbl_1[0], rtrans_1)) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 1\n");
- }
- // teardown
- trans_tbl_destroy(&tbl_1[0]);
- trans_tbl_destroy(&tbl_1[1]);
- free(rtrans_1);
- if (verbose) printf("END test 1\n");
-
- if (success == NUM_TESTS) {
- return 0;
- } else {
- fprintf(stderr, "%d failures %d successes\n", failure, success);
- return 1;
- }
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* test/merge/test_rtrans_build.c -- header translation test harness.
-
- Copyright (C) 2013, 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_sort.c"
-
-void dump_rtrans(int* rtrans, int n, int n_targets) {
- fprintf(samtools_stdout, "->n_targets:(%d)\n", n_targets);
- int i, j;
- for (i = 0; i < n; ++i) {
- fprintf(samtools_stderr, "%d",rtrans[i*n_targets+0]);
- for (j = 1; j < n_targets; ++j)
- fprintf(samtools_stderr, "\t%d",rtrans[i*n_targets+j]);
- fprintf(samtools_stderr, "\n");
- }
-}
-
-void setup_test_1(trans_tbl_t* tbl) {
- tbl[0].n_targets = 2;
- tbl[0].tid_trans = calloc(sizeof(int), 2);
- tbl[0].tid_trans[0] = 0;
- tbl[0].tid_trans[1] = 1;
- tbl[0].rg_trans = kh_init(c2c);
- tbl[0].pg_trans = kh_init(c2c);
-
- tbl[1].n_targets = 2;
- tbl[1].tid_trans = calloc(sizeof(int), 2);
- tbl[1].tid_trans[0] = 1;
- tbl[1].tid_trans[1] = 2;
- tbl[1].rg_trans = kh_init(c2c);
- tbl[1].pg_trans = kh_init(c2c);
-}
-
-bool check_test_1(trans_tbl_t* tbl, int* rtrans) {
- // Check input is unchanged
-
- // Check output
-
- return true;
-}
-
-
-int samtools_test_rtrans_build_main(int argc, char**argv)
-{
- const int NUM_TESTS = 1;
- int verbose = 0;
- int success = 0;
- int failure = 0;
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- break;
- }
- }
- const long GIMMICK_SEED = 0x1234330e;
- srand48(GIMMICK_SEED);
-
- if (verbose) fprintf(samtools_stdout, "BEGIN test 1\n");
- // setup
- trans_tbl_t tbl_1[2];
- int n_targets_1 = 3;
- int n_1 = 2;
- int* rtrans_1 = NULL;
- setup_test_1(&tbl_1[0]);
- // test
- if (verbose > 1) {
- // dump_trans_tid
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 1\n");
- rtrans_1 = rtrans_build(n_1, n_targets_1, &tbl_1[0]);
- if (verbose) fprintf(samtools_stdout, "END RUN test 1\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "rtrans\n");
- dump_rtrans(rtrans_1, n_1, n_targets_1);
- }
- if (check_test_1(&tbl_1[0], rtrans_1)) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 1\n");
- }
- // teardown
- trans_tbl_destroy(&tbl_1[0]);
- trans_tbl_destroy(&tbl_1[1]);
- free(rtrans_1);
- if (verbose) fprintf(samtools_stdout, "END test 1\n");
-
- if (success == NUM_TESTS) {
- return 0;
- } else {
- fprintf(samtools_stderr, "%d failures %d successes\n", failure, success);
- return 1;
- }
-}
+++ /dev/null
-/* test/merge/test_trans_tbl_init.c -- merge test harness.
-
- Copyright (C) 2013, 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_sort.c"
-#include <assert.h>
-#include <regex.h>
-
-typedef struct refseq_info {
- const char *name;
- uint32_t len;
-} refseq_info_t;
-
-void dump_header(bam_hdr_t* hdr) {
- printf("->n_targets:(%d)\n", hdr->n_targets);
- int i;
- for (i = 0; i < hdr->n_targets; ++i) {
- printf("->target_name[%d]:(%s)\n",i,hdr->target_name[i]);
- printf("->target_len[%d]:(%d)\n",i,hdr->target_len[i]);
- }
-
- printf("->text:(");
- fwrite((void*)hdr->text, (size_t) hdr->l_text, 1, stdout);
- printf(")\n");
-}
-
-static int populate_merged_header(bam_hdr_t *hdr, merged_header_t *merged_hdr) {
- trans_tbl_t dummy;
- int res;
- res = trans_tbl_init(merged_hdr, hdr, &dummy, 0, 0, 1, NULL);
- trans_tbl_destroy(&dummy);
- return res;
-}
-
-/*
- * Populate merged_hdr with data from bam0_header_text and bam0_refseqs.
- * Return bam_hdr_t based on the content in bam1_header_text and bam1_refseqs.
- */
-
-bam_hdr_t * setup_test(const char *bam0_header_text,
- const refseq_info_t *bam0_refseqs,
- int32_t bam0_n_refseqs,
- const char *bam1_header_text,
- const refseq_info_t *bam1_refseqs,
- int32_t bam1_n_refseqs,
- merged_header_t *merged_hdr) {
- bam_hdr_t* bam0 = NULL;
- bam_hdr_t* bam1 = NULL;
- int32_t i;
-
- bam0 = bam_hdr_init();
- bam0->text = strdup(bam0_header_text);
- if (!bam0->text) goto fail;
- bam0->l_text = strlen(bam0_header_text);
- bam0->n_targets = 1;
- bam0->target_name = (char**)calloc(bam0_n_refseqs, sizeof(char*));
- bam0->target_len = (uint32_t*)calloc(bam0_n_refseqs, sizeof(uint32_t));
- for (i = 0; i < bam0_n_refseqs; i++) {
- bam0->target_name[i] = strdup(bam0_refseqs[i].name);
- if (!bam0->target_name[i]) goto fail;
- bam0->target_len[i] = bam0_refseqs[i].len;
- }
-
- if (populate_merged_header(bam0, merged_hdr)) goto fail;
-
- bam1 = bam_hdr_init();
- if (!bam1) goto fail;
- bam1->text = strdup(bam1_header_text);
- if (!bam1->text) goto fail;
- bam1->l_text = strlen(bam1_header_text);
- bam1->n_targets = bam1_n_refseqs;
- bam1->target_name = (char**)calloc(bam1_n_refseqs, sizeof(char*));
- bam1->target_len = (uint32_t*)calloc(bam1_n_refseqs, sizeof(uint32_t));
- for (i = 0; i < bam1_n_refseqs; i++) {
- bam1->target_name[i] = strdup(bam1_refseqs[i].name);
- if (!bam1->target_name[i]) goto fail;
- bam1->target_len[i] = bam1_refseqs[i].len;
- }
-
- bam_hdr_destroy(bam0);
- return bam1;
-
- fail:
- bam_hdr_destroy(bam1);
- bam_hdr_destroy(bam0);
- return NULL;
-}
-
-#define NELE(x) (sizeof((x)) / sizeof((x)[0]))
-
-static const char init_text[] =
- "@HD\tVN:1.4\tSO:unknown\n"
- "@SQ\tSN:fish\tLN:133\tSP:frog";
-
-static const refseq_info_t init_refs[1] = {
- { "fish", 133 }
-};
-
-static const char test_1_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:fish\tLN:133\n";
-
-static const refseq_info_t test_1_refs[1] = {
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_1(merged_header_t *merged_hdr) {
- return setup_test(init_text, init_refs, NELE(init_refs),
- test_1_trans_text, test_1_refs, NELE(test_1_refs),
- merged_hdr);
-}
-
-bool check_test_1(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_1_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen( test_1_trans_text)
- || translate->n_targets != 1
- ) return false;
-
- // Check output header
- const char out_regex[] =
- "^@HD\tVN:1.4\tSO:unknown\n"
- "@SQ\tSN:fish\tLN:133\tSP:frog\n$";
-
- regex_t check_regex;
- regcomp(&check_regex, out_regex, REG_EXTENDED|REG_NOSUB);
-
- if ( regexec(&check_regex, out->text, 0, NULL, 0) != 0 || out->n_targets != 1 ) return false;
-
- regfree(&check_regex);
-
- // Check output tbl
- if (tbl[0].n_targets != 1 || tbl[0].tid_trans[0] != 0 || tbl[0].lost_coord_sort) return false;
-
- return true;
-}
-
-static const char test_2_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:donkey\tLN:133\n"
-"@SQ\tSN:fish\tLN:133";
-
-static const refseq_info_t test_2_refs[2] = {
- { "donkey", 133 },
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_2(merged_header_t *merged_hdr) {
- return setup_test(init_text, init_refs, NELE(init_refs),
- test_2_trans_text, test_2_refs, NELE(test_2_refs),
- merged_hdr);
-}
-
-bool check_test_2(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_2_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen(test_2_trans_text)
- || translate->n_targets != 2
- ) return false;
-
- // Check output header
- const char out_regex[] =
- "^@HD\tVN:1.4\tSO:unknown\n"
- "@SQ\tSN:fish\tLN:133\tSP:frog\n"
- "@SQ\tSN:donkey\tLN:133\n$";
-
- regex_t check_regex;
- regcomp(&check_regex, out_regex, REG_EXTENDED|REG_NOSUB);
-
- if ( regexec(&check_regex, out->text, 0, NULL, 0) != 0 || out->n_targets != 2 ) return false;
-
- regfree(&check_regex);
-
- // Check output tbl
- if (tbl[0].n_targets != 2 || tbl[0].tid_trans[0] != 1 || tbl[0].tid_trans[1] != 0) return false;
-
- return true;
-}
-
-static const char test_3_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:donkey\tLN:133\n"
-"@SQ\tSN:fish\tLN:133\n"
-"@RG\tID:fish\tPU:trans\n";
-
-static const refseq_info_t test_3_refs[2] = {
- { "donkey", 133 },
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_3(merged_header_t *merged_hdr) {
- return setup_test(init_text, init_refs, NELE(init_refs),
- test_3_trans_text, test_3_refs, NELE(test_3_refs),
- merged_hdr);
-}
-
-bool check_test_3(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_3_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen(test_3_trans_text)
- || translate->n_targets != 2
- ) return false;
- return true;
-}
-
-static const char test_4_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:donkey\tLN:133\n"
-"@SQ\tSN:fish\tLN:133\n"
-"@RG\tID:fish\tPU:trans\n";
-
-static const refseq_info_t test_4_refs[2] = {
- { "donkey", 133 },
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_4(merged_header_t *merged_hdr) {
- const char* t4_init_text =
- "@HD\tVN:1.4\tSO:unknown\n"
- "@SQ\tSN:fish\tLN:133\tSP:frog\n"
- "@RG\tID:fish\tPU:out\n";
-
- return setup_test(t4_init_text, init_refs, NELE(init_refs),
- test_4_trans_text, test_4_refs, NELE(test_4_refs),
- merged_hdr);
-}
-
-bool check_test_4(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_4_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen(test_4_trans_text)
- || translate->n_targets != 2
- ) return false;
- return true;
-}
-
-static const char test_5_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:donkey\tLN:133\n"
-"@SQ\tSN:fish\tLN:133\n"
-"@RG\tID:fish\tPU:trans\n"
-"@PG\tXX:dummy\tID:fish\tDS:trans\n"
-"@PG\tPP:fish\tID:hook\tDS:trans\n";
-
-static const refseq_info_t test_5_refs[2] = {
- { "donkey", 133 },
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_5(merged_header_t *merged_hdr) {
- const char* t5_init_text =
- "@HD\tVN:1.4\tSO:unknown\n"
- "@SQ\tSN:fish\tLN:133\tSP:frog\n"
- "@RG\tID:fish\tPU:out\n"
- "@PG\tXX:dummyx\tID:fish\tDS:out\n"
- "@PG\tPP:fish\tID:hook\tDS:out\n";
-
- return setup_test(t5_init_text, init_refs, NELE(init_refs),
- test_5_trans_text, test_5_refs, NELE(test_5_refs),
- merged_hdr);
-}
-
-bool check_test_5(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_5_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen(test_5_trans_text)
- || translate->n_targets != 2
- ) return false;
- return true;
-}
-
-static const char test_6_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:donkey\tLN:133\n"
-"@SQ\tSN:fish\tLN:133\n"
-"@RG\tID:fish\tPU:trans\n"
-"@PG\tXX:dummy\tID:fish\tDS:trans\n"
-"@PG\tPP:fish\tID:hook\tDS:trans\n";
-
-static const refseq_info_t test_6_refs[2] = {
- { "donkey", 133 },
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_6(merged_header_t *merged_hdr) {
- return setup_test(init_text, init_refs, NELE(init_refs),
- test_6_trans_text, test_6_refs, NELE(test_6_refs),
- merged_hdr);
-}
-
-bool check_test_6(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_6_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen(test_5_trans_text)
- || translate->n_targets != 2
- ) return false;
- return true;
-}
-
-int main(int argc, char**argv)
-{
- const int NUM_TESTS = 6;
- int verbose = 0;
- int success = 0;
- int failure = 0;
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- break;
- }
- }
-
- // Set the seed to a fixed value so that calls to lrand48 within functions return predictable values
- const long GIMMICK_SEED = 0x1234330e;
- srand48(GIMMICK_SEED);
-
- bam_hdr_t* out;
- bam_hdr_t* translate;
-
- if (verbose) printf("BEGIN test 1\n");
- // setup
- trans_tbl_t tbl_1;
- merged_header_t *merged_hdr = init_merged_header();
- translate = setup_test_1(merged_hdr);
- assert(translate);
- // test
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- }
- if (verbose) printf("RUN test 1\n");
- trans_tbl_init(merged_hdr, translate, &tbl_1, false, false, true, NULL);
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) printf("END RUN test 1\n");
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- printf("out\n");
- dump_header(out);
- }
- if (check_test_1(translate, out, &tbl_1)) {
- if (verbose) printf("Test 1 : PASS\n");
- ++success;
- } else {
- if (verbose) printf("Test 1 : FAIL\n");
- fprintf(stderr, "Test 1 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_1);
- if (verbose) printf("END test 1\n");
-
- // test
- if (verbose) printf("BEGIN test 2\n");
- // reinit
- trans_tbl_t tbl_2;
-
- merged_hdr = init_merged_header();
- translate = setup_test_2(merged_hdr);
- assert(translate);
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- }
- if (verbose) printf("RUN test 2\n");
- trans_tbl_init(merged_hdr, translate, &tbl_2, false, false, true, NULL);
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) printf("END RUN test 2\n");
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- printf("out\n");
- dump_header(out);
- }
- if (check_test_2(translate, out, &tbl_2)) {
- if (verbose) printf("Test 2 : PASS\n");
- ++success;
- } else {
- if (verbose) printf("Test 2 : FAIL\n");
- fprintf(stderr, "Test 2 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_2);
- if (verbose) printf("END test 2\n");
-
- // test
- if (verbose) printf("BEGIN test 3\n");
- // reinit
- trans_tbl_t tbl_3;
- merged_hdr = init_merged_header();
- translate = setup_test_3(merged_hdr);
- assert(translate);
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- }
- if (verbose) printf("RUN test 3\n");
- trans_tbl_init(merged_hdr, translate, &tbl_3, false, false, true, NULL);
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) printf("END RUN test 3\n");
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- printf("out\n");
- dump_header(out);
- }
- if (check_test_3(translate, out, &tbl_3)) {
- if (verbose) printf("Test 3 : PASS\n");
- ++success;
- } else {
- if (verbose) printf("Test 3 : FAIL\n");
- fprintf(stderr, "Test 3 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_3);
- if (verbose) printf("END test 3\n");
-
- // test
- if (verbose) printf("BEGIN test 4\n");
- // reinit
- trans_tbl_t tbl_4;
- merged_hdr = init_merged_header();
- translate = setup_test_4(merged_hdr);
- assert(translate);
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- }
- if (verbose) printf("RUN test 4\n");
- trans_tbl_init(merged_hdr, translate, &tbl_4, false, false, true, NULL);
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) printf("END RUN test 4\n");
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- printf("out\n");
- dump_header(out);
- }
- if (check_test_4(translate, out, &tbl_4)) {
- if (verbose) printf("Test 4 : PASS\n");
- ++success;
- } else {
- if (verbose) printf("Test 4 : FAIL\n");
- fprintf(stderr, "Test 4 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_4);
- if (verbose) printf("END test 4\n");
-
- // test
- if (verbose) printf("BEGIN test 5\n");
- // reinit
- trans_tbl_t tbl_5;
- merged_hdr = init_merged_header();
- translate = setup_test_5(merged_hdr);
- assert(translate);
- if (verbose > 1) {
-
- printf("translate\n");
- dump_header(translate);
- }
- if (verbose) printf("RUN test 5\n");
- trans_tbl_init(merged_hdr, translate, &tbl_5, false, false, true, NULL);
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) printf("END RUN test 5\n");
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- printf("out\n");
- dump_header(out);
- }
- if (check_test_5(translate, out, &tbl_5)) {
- if (verbose) printf("Test 5 : PASS\n");
- ++success;
- } else {
- if (verbose) printf("Test 5 : FAIL\n");
- fprintf(stderr, "Test 5 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_5);
- if (verbose) printf("END test 5\n");
-
- // test
- if (verbose) printf("BEGIN test 6\n");
- // reinit
- trans_tbl_t tbl_6;
- merged_hdr = init_merged_header();
- translate = setup_test_6(merged_hdr);
- assert(translate);
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- }
- if (verbose) printf("RUN test 6\n");
- trans_tbl_init(merged_hdr, translate, &tbl_6, false, false, true, "filename");
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) printf("END RUN test 6\n");
- if (verbose > 1) {
- printf("translate\n");
- dump_header(translate);
- printf("out\n");
- dump_header(out);
- }
- if (check_test_6(translate, out, &tbl_6)) {
- if (verbose) printf("Test 6 : PASS\n");
- ++success;
- } else {
- if (verbose) printf("Test 6 : FAIL\n");
- fprintf(stderr, "Test 6 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_6);
- if (verbose) printf("END test 6\n");
-
- if (success == NUM_TESTS) {
- return 0;
- } else {
- fprintf(stderr, "%d failures %d successes\n", failure, success);
- return 1;
- }
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* test/merge/test_trans_tbl_init.c -- merge test harness.
-
- Copyright (C) 2013, 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_sort.c"
-#include <assert.h>
-#include <regex.h>
-
-typedef struct refseq_info {
- const char *name;
- uint32_t len;
-} refseq_info_t;
-
-void dump_header(bam_hdr_t* hdr) {
- fprintf(samtools_stdout, "->n_targets:(%d)\n", hdr->n_targets);
- int i;
- for (i = 0; i < hdr->n_targets; ++i) {
- fprintf(samtools_stdout, "->target_name[%d]:(%s)\n",i,hdr->target_name[i]);
- fprintf(samtools_stdout, "->target_len[%d]:(%d)\n",i,hdr->target_len[i]);
- }
-
- fprintf(samtools_stdout, "->text:(");
- fwrite((void*)hdr->text, (size_t) hdr->l_text, 1, samtools_stdout);
- fprintf(samtools_stdout, ")\n");
-}
-
-static int populate_merged_header(bam_hdr_t *hdr, merged_header_t *merged_hdr) {
- trans_tbl_t dummy;
- int res;
- res = trans_tbl_init(merged_hdr, hdr, &dummy, 0, 0, 1, NULL);
- trans_tbl_destroy(&dummy);
- return res;
-}
-
-/*
- * Populate merged_hdr with data from bam0_header_text and bam0_refseqs.
- * Return bam_hdr_t based on the content in bam1_header_text and bam1_refseqs.
- */
-
-bam_hdr_t * setup_test(const char *bam0_header_text,
- const refseq_info_t *bam0_refseqs,
- int32_t bam0_n_refseqs,
- const char *bam1_header_text,
- const refseq_info_t *bam1_refseqs,
- int32_t bam1_n_refseqs,
- merged_header_t *merged_hdr) {
- bam_hdr_t* bam0 = NULL;
- bam_hdr_t* bam1 = NULL;
- int32_t i;
-
- bam0 = bam_hdr_init();
- bam0->text = strdup(bam0_header_text);
- if (!bam0->text) goto fail;
- bam0->l_text = strlen(bam0_header_text);
- bam0->n_targets = 1;
- bam0->target_name = (char**)calloc(bam0_n_refseqs, sizeof(char*));
- bam0->target_len = (uint32_t*)calloc(bam0_n_refseqs, sizeof(uint32_t));
- for (i = 0; i < bam0_n_refseqs; i++) {
- bam0->target_name[i] = strdup(bam0_refseqs[i].name);
- if (!bam0->target_name[i]) goto fail;
- bam0->target_len[i] = bam0_refseqs[i].len;
- }
-
- if (populate_merged_header(bam0, merged_hdr)) goto fail;
-
- bam1 = bam_hdr_init();
- if (!bam1) goto fail;
- bam1->text = strdup(bam1_header_text);
- if (!bam1->text) goto fail;
- bam1->l_text = strlen(bam1_header_text);
- bam1->n_targets = bam1_n_refseqs;
- bam1->target_name = (char**)calloc(bam1_n_refseqs, sizeof(char*));
- bam1->target_len = (uint32_t*)calloc(bam1_n_refseqs, sizeof(uint32_t));
- for (i = 0; i < bam1_n_refseqs; i++) {
- bam1->target_name[i] = strdup(bam1_refseqs[i].name);
- if (!bam1->target_name[i]) goto fail;
- bam1->target_len[i] = bam1_refseqs[i].len;
- }
-
- bam_hdr_destroy(bam0);
- return bam1;
-
- fail:
- bam_hdr_destroy(bam1);
- bam_hdr_destroy(bam0);
- return NULL;
-}
-
-#define NELE(x) (sizeof((x)) / sizeof((x)[0]))
-
-static const char init_text[] =
- "@HD\tVN:1.4\tSO:unknown\n"
- "@SQ\tSN:fish\tLN:133\tSP:frog";
-
-static const refseq_info_t init_refs[1] = {
- { "fish", 133 }
-};
-
-static const char test_1_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:fish\tLN:133\n";
-
-static const refseq_info_t test_1_refs[1] = {
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_1(merged_header_t *merged_hdr) {
- return setup_test(init_text, init_refs, NELE(init_refs),
- test_1_trans_text, test_1_refs, NELE(test_1_refs),
- merged_hdr);
-}
-
-bool check_test_1(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_1_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen( test_1_trans_text)
- || translate->n_targets != 1
- ) return false;
-
- // Check output header
- const char out_regex[] =
- "^@HD\tVN:1.4\tSO:unknown\n"
- "@SQ\tSN:fish\tLN:133\tSP:frog\n$";
-
- regex_t check_regex;
- regcomp(&check_regex, out_regex, REG_EXTENDED|REG_NOSUB);
-
- if ( regexec(&check_regex, out->text, 0, NULL, 0) != 0 || out->n_targets != 1 ) return false;
-
- regfree(&check_regex);
-
- // Check output tbl
- if (tbl[0].n_targets != 1 || tbl[0].tid_trans[0] != 0 || tbl[0].lost_coord_sort) return false;
-
- return true;
-}
-
-static const char test_2_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:donkey\tLN:133\n"
-"@SQ\tSN:fish\tLN:133";
-
-static const refseq_info_t test_2_refs[2] = {
- { "donkey", 133 },
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_2(merged_header_t *merged_hdr) {
- return setup_test(init_text, init_refs, NELE(init_refs),
- test_2_trans_text, test_2_refs, NELE(test_2_refs),
- merged_hdr);
-}
-
-bool check_test_2(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_2_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen(test_2_trans_text)
- || translate->n_targets != 2
- ) return false;
-
- // Check output header
- const char out_regex[] =
- "^@HD\tVN:1.4\tSO:unknown\n"
- "@SQ\tSN:fish\tLN:133\tSP:frog\n"
- "@SQ\tSN:donkey\tLN:133\n$";
-
- regex_t check_regex;
- regcomp(&check_regex, out_regex, REG_EXTENDED|REG_NOSUB);
-
- if ( regexec(&check_regex, out->text, 0, NULL, 0) != 0 || out->n_targets != 2 ) return false;
-
- regfree(&check_regex);
-
- // Check output tbl
- if (tbl[0].n_targets != 2 || tbl[0].tid_trans[0] != 1 || tbl[0].tid_trans[1] != 0) return false;
-
- return true;
-}
-
-static const char test_3_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:donkey\tLN:133\n"
-"@SQ\tSN:fish\tLN:133\n"
-"@RG\tID:fish\tPU:trans\n";
-
-static const refseq_info_t test_3_refs[2] = {
- { "donkey", 133 },
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_3(merged_header_t *merged_hdr) {
- return setup_test(init_text, init_refs, NELE(init_refs),
- test_3_trans_text, test_3_refs, NELE(test_3_refs),
- merged_hdr);
-}
-
-bool check_test_3(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_3_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen(test_3_trans_text)
- || translate->n_targets != 2
- ) return false;
- return true;
-}
-
-static const char test_4_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:donkey\tLN:133\n"
-"@SQ\tSN:fish\tLN:133\n"
-"@RG\tID:fish\tPU:trans\n";
-
-static const refseq_info_t test_4_refs[2] = {
- { "donkey", 133 },
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_4(merged_header_t *merged_hdr) {
- const char* t4_init_text =
- "@HD\tVN:1.4\tSO:unknown\n"
- "@SQ\tSN:fish\tLN:133\tSP:frog\n"
- "@RG\tID:fish\tPU:out\n";
-
- return setup_test(t4_init_text, init_refs, NELE(init_refs),
- test_4_trans_text, test_4_refs, NELE(test_4_refs),
- merged_hdr);
-}
-
-bool check_test_4(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_4_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen(test_4_trans_text)
- || translate->n_targets != 2
- ) return false;
- return true;
-}
-
-static const char test_5_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:donkey\tLN:133\n"
-"@SQ\tSN:fish\tLN:133\n"
-"@RG\tID:fish\tPU:trans\n"
-"@PG\tXX:dummy\tID:fish\tDS:trans\n"
-"@PG\tPP:fish\tID:hook\tDS:trans\n";
-
-static const refseq_info_t test_5_refs[2] = {
- { "donkey", 133 },
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_5(merged_header_t *merged_hdr) {
- const char* t5_init_text =
- "@HD\tVN:1.4\tSO:unknown\n"
- "@SQ\tSN:fish\tLN:133\tSP:frog\n"
- "@RG\tID:fish\tPU:out\n"
- "@PG\tXX:dummyx\tID:fish\tDS:out\n"
- "@PG\tPP:fish\tID:hook\tDS:out\n";
-
- return setup_test(t5_init_text, init_refs, NELE(init_refs),
- test_5_trans_text, test_5_refs, NELE(test_5_refs),
- merged_hdr);
-}
-
-bool check_test_5(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_5_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen(test_5_trans_text)
- || translate->n_targets != 2
- ) return false;
- return true;
-}
-
-static const char test_6_trans_text[] =
-"@HD\tVN:1.4\tSO:unknown\n"
-"@SQ\tSN:donkey\tLN:133\n"
-"@SQ\tSN:fish\tLN:133\n"
-"@RG\tID:fish\tPU:trans\n"
-"@PG\tXX:dummy\tID:fish\tDS:trans\n"
-"@PG\tPP:fish\tID:hook\tDS:trans\n";
-
-static const refseq_info_t test_6_refs[2] = {
- { "donkey", 133 },
- { "fish", 133 }
-};
-
-bam_hdr_t * setup_test_6(merged_header_t *merged_hdr) {
- return setup_test(init_text, init_refs, NELE(init_refs),
- test_6_trans_text, test_6_refs, NELE(test_6_refs),
- merged_hdr);
-}
-
-bool check_test_6(bam_hdr_t* translate, bam_hdr_t* out, trans_tbl_t* tbl) {
- // Check input is unchanged
- if (
- strncmp(test_6_trans_text, translate->text, translate->l_text)
- || translate->l_text != strlen(test_5_trans_text)
- || translate->n_targets != 2
- ) return false;
- return true;
-}
-
-int samtools_test_trans_tbl_init_main(int argc, char**argv)
-{
- const int NUM_TESTS = 6;
- int verbose = 0;
- int success = 0;
- int failure = 0;
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- break;
- }
- }
-
- // Set the seed to a fixed value so that calls to lrand48 within functions return predictable values
- const long GIMMICK_SEED = 0x1234330e;
- srand48(GIMMICK_SEED);
-
- bam_hdr_t* out;
- bam_hdr_t* translate;
-
- if (verbose) fprintf(samtools_stdout, "BEGIN test 1\n");
- // setup
- trans_tbl_t tbl_1;
- merged_header_t *merged_hdr = init_merged_header();
- translate = setup_test_1(merged_hdr);
- assert(translate);
- // test
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 1\n");
- trans_tbl_init(merged_hdr, translate, &tbl_1, false, false, true, NULL);
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) fprintf(samtools_stdout, "END RUN test 1\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- fprintf(samtools_stdout, "out\n");
- dump_header(out);
- }
- if (check_test_1(translate, out, &tbl_1)) {
- if (verbose) fprintf(samtools_stdout, "Test 1 : PASS\n");
- ++success;
- } else {
- if (verbose) fprintf(samtools_stdout, "Test 1 : FAIL\n");
- fprintf(samtools_stderr, "Test 1 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_1);
- if (verbose) fprintf(samtools_stdout, "END test 1\n");
-
- // test
- if (verbose) fprintf(samtools_stdout, "BEGIN test 2\n");
- // reinit
- trans_tbl_t tbl_2;
-
- merged_hdr = init_merged_header();
- translate = setup_test_2(merged_hdr);
- assert(translate);
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 2\n");
- trans_tbl_init(merged_hdr, translate, &tbl_2, false, false, true, NULL);
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) fprintf(samtools_stdout, "END RUN test 2\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- fprintf(samtools_stdout, "out\n");
- dump_header(out);
- }
- if (check_test_2(translate, out, &tbl_2)) {
- if (verbose) fprintf(samtools_stdout, "Test 2 : PASS\n");
- ++success;
- } else {
- if (verbose) fprintf(samtools_stdout, "Test 2 : FAIL\n");
- fprintf(samtools_stderr, "Test 2 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_2);
- if (verbose) fprintf(samtools_stdout, "END test 2\n");
-
- // test
- if (verbose) fprintf(samtools_stdout, "BEGIN test 3\n");
- // reinit
- trans_tbl_t tbl_3;
- merged_hdr = init_merged_header();
- translate = setup_test_3(merged_hdr);
- assert(translate);
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 3\n");
- trans_tbl_init(merged_hdr, translate, &tbl_3, false, false, true, NULL);
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) fprintf(samtools_stdout, "END RUN test 3\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- fprintf(samtools_stdout, "out\n");
- dump_header(out);
- }
- if (check_test_3(translate, out, &tbl_3)) {
- if (verbose) fprintf(samtools_stdout, "Test 3 : PASS\n");
- ++success;
- } else {
- if (verbose) fprintf(samtools_stdout, "Test 3 : FAIL\n");
- fprintf(samtools_stderr, "Test 3 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_3);
- if (verbose) fprintf(samtools_stdout, "END test 3\n");
-
- // test
- if (verbose) fprintf(samtools_stdout, "BEGIN test 4\n");
- // reinit
- trans_tbl_t tbl_4;
- merged_hdr = init_merged_header();
- translate = setup_test_4(merged_hdr);
- assert(translate);
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 4\n");
- trans_tbl_init(merged_hdr, translate, &tbl_4, false, false, true, NULL);
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) fprintf(samtools_stdout, "END RUN test 4\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- fprintf(samtools_stdout, "out\n");
- dump_header(out);
- }
- if (check_test_4(translate, out, &tbl_4)) {
- if (verbose) fprintf(samtools_stdout, "Test 4 : PASS\n");
- ++success;
- } else {
- if (verbose) fprintf(samtools_stdout, "Test 4 : FAIL\n");
- fprintf(samtools_stderr, "Test 4 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_4);
- if (verbose) fprintf(samtools_stdout, "END test 4\n");
-
- // test
- if (verbose) fprintf(samtools_stdout, "BEGIN test 5\n");
- // reinit
- trans_tbl_t tbl_5;
- merged_hdr = init_merged_header();
- translate = setup_test_5(merged_hdr);
- assert(translate);
- if (verbose > 1) {
-
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 5\n");
- trans_tbl_init(merged_hdr, translate, &tbl_5, false, false, true, NULL);
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) fprintf(samtools_stdout, "END RUN test 5\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- fprintf(samtools_stdout, "out\n");
- dump_header(out);
- }
- if (check_test_5(translate, out, &tbl_5)) {
- if (verbose) fprintf(samtools_stdout, "Test 5 : PASS\n");
- ++success;
- } else {
- if (verbose) fprintf(samtools_stdout, "Test 5 : FAIL\n");
- fprintf(samtools_stderr, "Test 5 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_5);
- if (verbose) fprintf(samtools_stdout, "END test 5\n");
-
- // test
- if (verbose) fprintf(samtools_stdout, "BEGIN test 6\n");
- // reinit
- trans_tbl_t tbl_6;
- merged_hdr = init_merged_header();
- translate = setup_test_6(merged_hdr);
- assert(translate);
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 6\n");
- trans_tbl_init(merged_hdr, translate, &tbl_6, false, false, true, "filename");
- out = finish_merged_header(merged_hdr);
- free_merged_header(merged_hdr);
- if (verbose) fprintf(samtools_stdout, "END RUN test 6\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "translate\n");
- dump_header(translate);
- fprintf(samtools_stdout, "out\n");
- dump_header(out);
- }
- if (check_test_6(translate, out, &tbl_6)) {
- if (verbose) fprintf(samtools_stdout, "Test 6 : PASS\n");
- ++success;
- } else {
- if (verbose) fprintf(samtools_stdout, "Test 6 : FAIL\n");
- fprintf(samtools_stderr, "Test 6 : FAIL\n");
- ++failure;
- }
- // teardown
- bam_hdr_destroy(translate);
- bam_hdr_destroy(out);
- trans_tbl_destroy(&tbl_6);
- if (verbose) fprintf(samtools_stdout, "END test 6\n");
-
- if (success == NUM_TESTS) {
- return 0;
- } else {
- fprintf(samtools_stderr, "%d failures %d successes\n", failure, success);
- return 1;
- }
-}
+++ /dev/null
-/* test/split/test_count_rg.c -- split test cases.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_split.c"
-#include "../test.h"
-#include <stdlib.h>
-#include <unistd.h>
-
-void setup_test_1(bam_hdr_t** hdr_in)
-{
- *hdr_in = bam_hdr_init();
- const char *test1 =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@RG\tID:fish\n";
- (*hdr_in)->text = strdup(test1);
- (*hdr_in)->l_text = strlen(test1);
-}
-
-int main(int argc, char**argv)
-{
- // test state
- const int NUM_TESTS = 1;
- int verbose = 0;
- int success = 0;
- int failure = 0;
-
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- printf(
- "usage: test_count_rg [-v]\n\n"
- " -v verbose output\n"
- );
- break;
- }
- }
-
-
- // Setup stderr redirect
- kstring_t res = { 0, 0, NULL };
- FILE* orig_stderr = fdopen(dup(STDERR_FILENO), "a"); // Save stderr
- char* tempfname = (optind < argc)? argv[optind] : "test_count_rg.tmp";
- FILE* check = NULL;
-
- // setup
- if (verbose) printf("BEGIN test 1\n"); // TID test
- bam_hdr_t* hdr1;
- size_t count;
- char** output;
- setup_test_1(&hdr1);
- if (verbose > 1) {
- printf("hdr1\n");
- dump_hdr(hdr1);
- }
- if (verbose) printf("RUN test 1\n");
-
- // test
- xfreopen(tempfname, "w", stderr); // Redirect stderr to pipe
- bool result_1 = count_RG(hdr1, &count, &output);
- fclose(stderr);
-
- if (verbose) printf("END RUN test 1\n");
- if (verbose > 1) {
- printf("b\n");
- dump_hdr(hdr1);
- }
-
- // check result
- check = fopen(tempfname, "r");
- if (result_1 && count == 1 && !strcmp(output[0], "fish")
- && kgetline(&res, (kgets_func *)fgets, check) < 0
- && (feof(check) || res.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 1\n");
- }
- fclose(check);
-
- // teardown
- int i;
- for (i = 0; i < count; i++){
- free(output[i]);
- }
- free(output);
- bam_hdr_destroy(hdr1);
- if (verbose) printf("END test 1\n");
-
- // Cleanup
- free(res.s);
- remove(tempfname);
- if (failure > 0)
- fprintf(orig_stderr, "%d failures %d successes\n", failure, success);
- fclose(orig_stderr);
-
- return (success == NUM_TESTS)? EXIT_SUCCESS : EXIT_FAILURE;
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* test/split/test_count_rg.c -- split test cases.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_split.c"
-#include "../test.h"
-#include <stdlib.h>
-#include <unistd.h>
-
-void setup_test_1(bam_hdr_t** hdr_in)
-{
- *hdr_in = bam_hdr_init();
- const char *test1 =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@RG\tID:fish\n";
- (*hdr_in)->text = strdup(test1);
- (*hdr_in)->l_text = strlen(test1);
-}
-
-int samtools_test_count_rg_main(int argc, char**argv)
-{
- // test state
- const int NUM_TESTS = 1;
- int verbose = 0;
- int success = 0;
- int failure = 0;
-
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- fprintf(samtools_stdout,
- "usage: test_count_rg [-v]\n\n"
- " -v verbose output\n"
- );
- break;
- }
- }
-
-
- // Setup samtools_stderr redirect
- kstring_t res = { 0, 0, NULL };
- FILE* orig_samtools_stderr = fdopen(dup(STDERR_FILENO), "a"); // Save samtools_stderr
- char* tempfname = (optind < argc)? argv[optind] : "test_count_rg.tmp";
- FILE* check = NULL;
-
- // setup
- if (verbose) fprintf(samtools_stdout, "BEGIN test 1\n"); // TID test
- bam_hdr_t* hdr1;
- size_t count;
- char** output;
- setup_test_1(&hdr1);
- if (verbose > 1) {
- fprintf(samtools_stdout, "hdr1\n");
- dump_hdr(hdr1);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 1\n");
-
- // test
- xfreopen(tempfname, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- bool result_1 = count_RG(hdr1, &count, &output);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(samtools_stdout, "END RUN test 1\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "b\n");
- dump_hdr(hdr1);
- }
-
- // check result
- check = fopen(tempfname, "r");
- if (result_1 && count == 1 && !strcmp(output[0], "fish")
- && kgetline(&res, (kgets_func *)fgets, check) < 0
- && (feof(check) || res.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 1\n");
- }
- fclose(check);
-
- // teardown
- int i;
- for (i = 0; i < count; i++){
- free(output[i]);
- }
- free(output);
- bam_hdr_destroy(hdr1);
- if (verbose) fprintf(samtools_stdout, "END test 1\n");
-
- // Cleanup
- free(res.s);
- remove(tempfname);
- if (failure > 0)
- fprintf(orig_samtools_stderr, "%d failures %d successes\n", failure, success);
- fclose(orig_samtools_stderr);
-
- return (success == NUM_TESTS)? EXIT_SUCCESS : EXIT_FAILURE;
-}
+++ /dev/null
-/* test/split/test_expand_format_string.c -- split format string test cases.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_split.c"
-#include "../test.h"
-#include <stdlib.h>
-#include <unistd.h>
-
-void setup_test_1(bam_hdr_t** hdr_in)
-{
- *hdr_in = bam_hdr_init();
- const char *test1 =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@RG\tID:fish\n";
- (*hdr_in)->text = strdup(test1);
- (*hdr_in)->l_text = strlen(test1);
-}
-
-int main(int argc, char**argv)
-{
- // test state
- const int NUM_TESTS = 1;
- int verbose = 0;
- int success = 0;
- int failure = 0;
-
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- printf(
- "usage: test_expand_format_string [-v]\n\n"
- " -v verbose output\n"
- );
- break;
- }
- }
-
-
- // Setup stderr redirect
- kstring_t res = { 0, 0, NULL };
- FILE* orig_stderr = fdopen(dup(STDERR_FILENO), "a"); // Save stderr
- char* tempfname = (optind < argc)? argv[optind] : "test_expand_format_string.tmp";
- FILE* check = NULL;
-
- // setup
- if (verbose) printf("BEGIN test 1\n"); // default format string test
- const char* format_string_1 = "%*_%#.bam";
- const char* basename_1 = "basename";
- const char* rg_id_1 = "1#2.3";
- const int rg_idx_1 = 4;
- if (verbose > 1) {
- printf("format_string:%s\n"
- "basename:%s\n"
- "rg_id:%s\n"
- "rg_idx:%d\n", format_string_1, basename_1, rg_id_1, rg_idx_1);
- }
- if (verbose) printf("RUN test 1\n");
-
- // test
- xfreopen(tempfname, "w", stderr); // Redirect stderr to pipe
- char* output_1 = expand_format_string(format_string_1, basename_1, rg_id_1, rg_idx_1, NULL);
- fclose(stderr);
-
- if (verbose) printf("END RUN test 1\n");
- if (verbose > 1) {
- printf("format_string:%s\n"
- "basename:%s\n"
- "rg_id:%s\n"
- "rg_idx:%d\n", format_string_1, basename_1, rg_id_1, rg_idx_1);
- }
-
- // check result
- res.l = 0;
- check = fopen(tempfname, "r");
- if (output_1 != NULL && !strcmp(output_1, "basename_4.bam")
- && kgetline(&res, (kgets_func *)fgets, check) < 0
- && (feof(check) || res.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 1\n");
- }
- fclose(check);
-
- // teardown
- free(output_1);
- if (verbose) printf("END test 1\n");
-
- // Cleanup test harness
- free(res.s);
- remove(tempfname);
- if (failure > 0)
- fprintf(orig_stderr, "%d failures %d successes\n", failure, success);
- fclose(orig_stderr);
-
- return (success == NUM_TESTS)? EXIT_SUCCESS : EXIT_FAILURE;
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* test/split/test_expand_format_string.c -- split format string test cases.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_split.c"
-#include "../test.h"
-#include <stdlib.h>
-#include <unistd.h>
-
-void setup_test_1(bam_hdr_t** hdr_in)
-{
- *hdr_in = bam_hdr_init();
- const char *test1 =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@RG\tID:fish\n";
- (*hdr_in)->text = strdup(test1);
- (*hdr_in)->l_text = strlen(test1);
-}
-
-int samtools_test_expand_format_string_main(int argc, char**argv)
-{
- // test state
- const int NUM_TESTS = 1;
- int verbose = 0;
- int success = 0;
- int failure = 0;
-
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- fprintf(samtools_stdout,
- "usage: test_expand_format_string [-v]\n\n"
- " -v verbose output\n"
- );
- break;
- }
- }
-
-
- // Setup samtools_stderr redirect
- kstring_t res = { 0, 0, NULL };
- FILE* orig_samtools_stderr = fdopen(dup(STDERR_FILENO), "a"); // Save samtools_stderr
- char* tempfname = (optind < argc)? argv[optind] : "test_expand_format_string.tmp";
- FILE* check = NULL;
-
- // setup
- if (verbose) fprintf(samtools_stdout, "BEGIN test 1\n"); // default format string test
- const char* format_string_1 = "%*_%#.bam";
- const char* basename_1 = "basename";
- const char* rg_id_1 = "1#2.3";
- const int rg_idx_1 = 4;
- if (verbose > 1) {
- fprintf(samtools_stdout, "format_string:%s\n"
- "basename:%s\n"
- "rg_id:%s\n"
- "rg_idx:%d\n", format_string_1, basename_1, rg_id_1, rg_idx_1);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 1\n");
-
- // test
- xfreopen(tempfname, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- char* output_1 = expand_format_string(format_string_1, basename_1, rg_id_1, rg_idx_1, NULL);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(samtools_stdout, "END RUN test 1\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "format_string:%s\n"
- "basename:%s\n"
- "rg_id:%s\n"
- "rg_idx:%d\n", format_string_1, basename_1, rg_id_1, rg_idx_1);
- }
-
- // check result
- res.l = 0;
- check = fopen(tempfname, "r");
- if (output_1 != NULL && !strcmp(output_1, "basename_4.bam")
- && kgetline(&res, (kgets_func *)fgets, check) < 0
- && (feof(check) || res.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 1\n");
- }
- fclose(check);
-
- // teardown
- free(output_1);
- if (verbose) fprintf(samtools_stdout, "END test 1\n");
-
- // Cleanup test harness
- free(res.s);
- remove(tempfname);
- if (failure > 0)
- fprintf(orig_samtools_stderr, "%d failures %d successes\n", failure, success);
- fclose(orig_samtools_stderr);
-
- return (success == NUM_TESTS)? EXIT_SUCCESS : EXIT_FAILURE;
-}
+++ /dev/null
-/* test/split/test_filter_header_rg.c -- split test cases.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_split.c"
-#include "../test.h"
-#include <unistd.h>
-
-void setup_test_1(bam_hdr_t** hdr_in)
-{
- *hdr_in = bam_hdr_init();
- const char *test1 =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@RG\tID:fish\n";
- (*hdr_in)->text = strdup(test1);
- (*hdr_in)->l_text = strlen(test1);
-}
-
-bool check_test_1(const bam_hdr_t* hdr) {
- const char *test1_res =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@PG\tID:samtools\tPN:samtools\tVN:x.y.test\tCL:test_filter_header_rg foo bar baz\n";
-
- if (strcmp(hdr->text, test1_res)) {
- return false;
- }
- return true;
-}
-
-void setup_test_2(bam_hdr_t** hdr_in)
-{
- *hdr_in = bam_hdr_init();
- const char *test2 =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@RG\tID:fish\n";
- (*hdr_in)->text = strdup(test2);
- (*hdr_in)->l_text = strlen(test2);
-}
-
-bool check_test_2(const bam_hdr_t* hdr) {
- const char *test2_res =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@RG\tID:fish\n"
- "@PG\tID:samtools\tPN:samtools\tVN:x.y.test\tCL:test_filter_header_rg foo bar baz\n";
-
- if (strcmp(hdr->text, test2_res)) {
- return false;
- }
- return true;
-}
-
-int main(int argc, char *argv[])
-{
- // test state
- const int NUM_TESTS = 2;
- int verbose = 0;
- int success = 0;
- int failure = 0;
-
- int getopt_char;
- char *test_argv[] = { "test_filter_header_rg", "foo\tbar", "baz" };
- char *arg_list = stringify_argv(3, test_argv);
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- printf(
- "usage: test_filter_header_rg [-v]\n\n"
- " -v verbose output\n"
- );
- break;
- }
- }
-
-
- // Setup stderr redirect
- kstring_t res = { 0, 0, NULL };
- FILE* orig_stderr = fdopen(dup(STDERR_FILENO), "a"); // Save stderr
- char* tempfname = (optind < argc)? argv[optind] : "test_count_rg.tmp";
- FILE* check = NULL;
-
- // setup
- if (verbose) printf("BEGIN test 1\n"); // test eliminating a tag that isn't there
- bam_hdr_t* hdr1;
- const char* id_to_keep_1 = "1#2.3";
- setup_test_1(&hdr1);
- if (verbose > 1) {
- printf("hdr1\n");
- dump_hdr(hdr1);
- }
- if (verbose) printf("RUN test 1\n");
-
- // test
- xfreopen(tempfname, "w", stderr); // Redirect stderr to pipe
- bool result_1 = filter_header_rg(hdr1, id_to_keep_1, arg_list);
- fclose(stderr);
-
- if (verbose) printf("END RUN test 1\n");
- if (verbose > 1) {
- printf("hdr1\n");
- dump_hdr(hdr1);
- }
-
- // check result
- res.l = 0;
- check = fopen(tempfname, "r");
- if ( result_1
- && check_test_1(hdr1)
- && kgetline(&res, (kgets_func *)fgets, check) < 0
- && (feof(check) || res.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 1\n");
- }
- fclose(check);
-
- // teardown
- bam_hdr_destroy(hdr1);
- if (verbose) printf("END test 1\n");
-
- if (verbose) printf("BEGIN test 2\n"); // test eliminating a tag that is there
- bam_hdr_t* hdr2;
- const char* id_to_keep_2 = "fish";
- setup_test_2(&hdr2);
- if (verbose > 1) {
- printf("hdr2\n");
- dump_hdr(hdr2);
- }
- if (verbose) printf("RUN test 2\n");
-
- // test
- xfreopen(tempfname, "w", stderr); // Redirect stderr to pipe
- bool result_2 = filter_header_rg(hdr2, id_to_keep_2, arg_list);
- fclose(stderr);
-
- if (verbose) printf("END RUN test 2\n");
- if (verbose > 1) {
- printf("hdr2\n");
- dump_hdr(hdr2);
- }
-
- // check result
- res.l = 0;
- check = fopen(tempfname, "r");
- if ( result_2
- && check_test_2(hdr2)
- && kgetline(&res, (kgets_func *)fgets, check) < 0
- && (feof(check) || res.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) printf("FAIL test 2\n");
- }
- fclose(check);
-
- // teardown
- bam_hdr_destroy(hdr2);
- if (verbose) printf("END test 2\n");
-
-
- // Cleanup
- free(res.s);
- free(arg_list);
- remove(tempfname);
- if (failure > 0)
- fprintf(orig_stderr, "%d failures %d successes\n", failure, success);
- fclose(orig_stderr);
-
- return (success == NUM_TESTS)? EXIT_SUCCESS : EXIT_FAILURE;
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* test/split/test_filter_header_rg.c -- split test cases.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_split.c"
-#include "../test.h"
-#include <unistd.h>
-
-void setup_test_1(bam_hdr_t** hdr_in)
-{
- *hdr_in = bam_hdr_init();
- const char *test1 =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@RG\tID:fish\n";
- (*hdr_in)->text = strdup(test1);
- (*hdr_in)->l_text = strlen(test1);
-}
-
-bool check_test_1(const bam_hdr_t* hdr) {
- const char *test1_res =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@PG\tID:samtools\tPN:samtools\tVN:x.y.test\tCL:test_filter_header_rg foo bar baz\n";
-
- if (strcmp(hdr->text, test1_res)) {
- return false;
- }
- return true;
-}
-
-void setup_test_2(bam_hdr_t** hdr_in)
-{
- *hdr_in = bam_hdr_init();
- const char *test2 =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@RG\tID:fish\n";
- (*hdr_in)->text = strdup(test2);
- (*hdr_in)->l_text = strlen(test2);
-}
-
-bool check_test_2(const bam_hdr_t* hdr) {
- const char *test2_res =
- "@HD\tVN:1.4\n"
- "@SQ\tSN:blah\n"
- "@RG\tID:fish\n"
- "@PG\tID:samtools\tPN:samtools\tVN:x.y.test\tCL:test_filter_header_rg foo bar baz\n";
-
- if (strcmp(hdr->text, test2_res)) {
- return false;
- }
- return true;
-}
-
-int samtools_test_filter_header_rg_main(int argc, char *argv[])
-{
- // test state
- const int NUM_TESTS = 2;
- int verbose = 0;
- int success = 0;
- int failure = 0;
-
- int getopt_char;
- char *test_argv[] = { "test_filter_header_rg", "foo\tbar", "baz" };
- char *arg_list = stringify_argv(3, test_argv);
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- fprintf(samtools_stdout,
- "usage: test_filter_header_rg [-v]\n\n"
- " -v verbose output\n"
- );
- break;
- }
- }
-
-
- // Setup samtools_stderr redirect
- kstring_t res = { 0, 0, NULL };
- FILE* orig_samtools_stderr = fdopen(dup(STDERR_FILENO), "a"); // Save samtools_stderr
- char* tempfname = (optind < argc)? argv[optind] : "test_count_rg.tmp";
- FILE* check = NULL;
-
- // setup
- if (verbose) fprintf(samtools_stdout, "BEGIN test 1\n"); // test eliminating a tag that isn't there
- bam_hdr_t* hdr1;
- const char* id_to_keep_1 = "1#2.3";
- setup_test_1(&hdr1);
- if (verbose > 1) {
- fprintf(samtools_stdout, "hdr1\n");
- dump_hdr(hdr1);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 1\n");
-
- // test
- xfreopen(tempfname, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- bool result_1 = filter_header_rg(hdr1, id_to_keep_1, arg_list);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(samtools_stdout, "END RUN test 1\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "hdr1\n");
- dump_hdr(hdr1);
- }
-
- // check result
- res.l = 0;
- check = fopen(tempfname, "r");
- if ( result_1
- && check_test_1(hdr1)
- && kgetline(&res, (kgets_func *)fgets, check) < 0
- && (feof(check) || res.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 1\n");
- }
- fclose(check);
-
- // teardown
- bam_hdr_destroy(hdr1);
- if (verbose) fprintf(samtools_stdout, "END test 1\n");
-
- if (verbose) fprintf(samtools_stdout, "BEGIN test 2\n"); // test eliminating a tag that is there
- bam_hdr_t* hdr2;
- const char* id_to_keep_2 = "fish";
- setup_test_2(&hdr2);
- if (verbose > 1) {
- fprintf(samtools_stdout, "hdr2\n");
- dump_hdr(hdr2);
- }
- if (verbose) fprintf(samtools_stdout, "RUN test 2\n");
-
- // test
- xfreopen(tempfname, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- bool result_2 = filter_header_rg(hdr2, id_to_keep_2, arg_list);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(samtools_stdout, "END RUN test 2\n");
- if (verbose > 1) {
- fprintf(samtools_stdout, "hdr2\n");
- dump_hdr(hdr2);
- }
-
- // check result
- res.l = 0;
- check = fopen(tempfname, "r");
- if ( result_2
- && check_test_2(hdr2)
- && kgetline(&res, (kgets_func *)fgets, check) < 0
- && (feof(check) || res.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(samtools_stdout, "FAIL test 2\n");
- }
- fclose(check);
-
- // teardown
- bam_hdr_destroy(hdr2);
- if (verbose) fprintf(samtools_stdout, "END test 2\n");
-
-
- // Cleanup
- free(res.s);
- free(arg_list);
- remove(tempfname);
- if (failure > 0)
- fprintf(orig_samtools_stderr, "%d failures %d successes\n", failure, success);
- fclose(orig_samtools_stderr);
-
- return (success == NUM_TESTS)? EXIT_SUCCESS : EXIT_FAILURE;
-}
+++ /dev/null
-/* test/split/test_parse_args.c -- split test cases.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_split.c"
-#include "../test.h"
-#include <stdlib.h>
-#include <unistd.h>
-
-void setup_test_1(int* argc, char*** argv)
-{
- *argc = 1;
- *argv = (char**)calloc(sizeof(char*), 1);
- (*argv)[0] = strdup("prog_name");
-}
-
-bool check_test_1(const parsed_opts_t* opts) {
- if ( opts->merged_input_name != NULL
- || opts->unaccounted_header_name != NULL
- || opts->unaccounted_name != NULL
- || strcmp(opts->output_format_string,"%*_%#.%.")
- || opts->verbose == true )
- return false;
- return true;
-}
-
-void setup_test_2(int* argc, char*** argv)
-{
- *argc = 2;
- *argv = (char**)calloc(sizeof(char*), 2);
- (*argv)[0] = strdup("prog_name");
- (*argv)[1] = strdup("merged.bam");
-}
-
-bool check_test_2(const parsed_opts_t* opts) {
- if ( opts->merged_input_name == NULL
- || strcmp(opts->merged_input_name, "merged.bam")
- || opts->unaccounted_header_name != NULL
- || opts->unaccounted_name != NULL
- || strcmp(opts->output_format_string,"%*_%#.%.")
- || opts->verbose == true )
- return false;
- return true;
-}
-
-int main(int argc, char**argv)
-{
- // test state
- const int NUM_TESTS = 2;
- int verbose = 0;
- int success = 0;
- int failure = 0;
-
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- printf(
- "usage: test_parse_args [-v]\n\n"
- " -v verbose output\n"
- );
- break;
- }
- }
-
- // Setup stdout and stderr redirect
- kstring_t res_stdout = { 0, 0, NULL };
- kstring_t res_stderr = { 0, 0, NULL };
- FILE* orig_stdout = fdopen(dup(STDOUT_FILENO), "a"); // Save stderr
- FILE* orig_stderr = fdopen(dup(STDERR_FILENO), "a"); // Save stderr
- char* tempfname_stdout = (optind < argc)? argv[optind] : "test_parse_args.tmp.o";
- char* tempfname_stderr = (optind < argc)? argv[optind] : "test_parse_args.tmp.e";
- FILE* check_stdout = NULL;
- FILE* check_stderr = NULL;
-
- // Cleanup getopt
- optind = 1;
-
- // setup
- if (verbose) fprintf(orig_stdout,"BEGIN test 1\n"); // test eliminating a tag that isn't there
- int argc_1;
- char** argv_1;
- setup_test_1(&argc_1, &argv_1);
- if (verbose > 1) {
- fprintf(orig_stdout, "argc: %d\n", argc_1);
- }
- if (verbose) fprintf(orig_stdout,"RUN test 1\n");
-
- // test
- xfreopen(tempfname_stdout, "w", stdout); // Redirect stdout to pipe
- xfreopen(tempfname_stderr, "w", stderr); // Redirect stderr to pipe
- parsed_opts_t* result_1 = parse_args(argc_1, argv_1);
- fclose(stdout);
- fclose(stderr);
-
- if (verbose) fprintf(orig_stdout, "END RUN test 1\n");
- if (verbose > 1) {
- fprintf(orig_stdout, "argc: %d\n", argc_1);
- }
-
- // check result
- res_stdout.l = res_stderr.l = 0;
- check_stdout = fopen(tempfname_stdout, "r");
- check_stderr = fopen(tempfname_stderr, "r");
- if ( !result_1
- && kgetline(&res_stdout, (kgets_func *)fgets, check_stdout) >= 0
- && !feof(check_stdout)
- && res_stdout.l > 0
- && kgetline(&res_stderr, (kgets_func *)fgets, check_stderr) < 0
- && (feof(check_stderr) || res_stderr.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(orig_stdout, "FAIL test 1\n");
- }
- fclose(check_stderr);
- fclose(check_stdout);
-
- // teardown
- cleanup_opts(result_1);
- int i = 0;
- for (i = 0; i < argc_1; ++i) {
- free(argv_1[i]);
- }
- free(argv_1);
- if (verbose) fprintf(orig_stdout, "END test 1\n");
-
- // Cleanup getopt
- optind = 1;
-
- if (verbose) fprintf(orig_stdout, "BEGIN test 2\n"); // test eliminating a tag that is there
- int argc_2;
- char** argv_2;
- setup_test_2(&argc_2, &argv_2);
- if (verbose > 1) {
- fprintf(orig_stdout, "argc: %d\n", argc_2);
- }
- if (verbose) fprintf(orig_stdout, "RUN test 2\n");
-
- // test
- xfreopen(tempfname_stdout, "w", stdout); // Redirect stdout to pipe
- xfreopen(tempfname_stderr, "w", stderr); // Redirect stderr to pipe
- parsed_opts_t* result_2 = parse_args(argc_2, argv_2);
- fclose(stdout);
- fclose(stderr);
-
- if (verbose) fprintf(orig_stdout, "END RUN test 2\n");
- if (verbose > 1) {
- fprintf(orig_stdout, "argc: %d\n", argc_2);
- }
-
- // check result
- res_stdout.l = res_stderr.l = 0;
- check_stdout = fopen(tempfname_stdout, "r");
- check_stderr = fopen(tempfname_stderr, "r");
- if ( result_2
- && check_test_2(result_2)
- && kgetline(&res_stdout, (kgets_func *)fgets, check_stdout) < 0
- && (feof(check_stdout) || res_stdout.l == 0)
- && kgetline(&res_stderr, (kgets_func *)fgets, check_stderr) < 0
- && (feof(check_stderr) || res_stderr.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(orig_stdout, "FAIL test 2\n");
- }
- fclose(check_stdout);
- fclose(check_stderr);
-
- // teardown
- cleanup_opts(result_2);
- int j = 0;
- for (j = 0; j < argc_2; ++j) {
- free(argv_2[j]);
- }
- free(argv_2);
-
- if (verbose) fprintf(orig_stdout, "END test 2\n");
-
-
- // Cleanup
- free(res_stdout.s);
- free(res_stderr.s);
- remove(tempfname_stdout);
- remove(tempfname_stderr);
- fclose(orig_stdout);
- if (failure > 0)
- fprintf(orig_stderr, "%d failures %d successes\n", failure, success);
- fclose(orig_stderr);
-
- return (success == NUM_TESTS)? EXIT_SUCCESS : EXIT_FAILURE;
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* test/split/test_parse_args.c -- split test cases.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_split.c"
-#include "../test.h"
-#include <stdlib.h>
-#include <unistd.h>
-
-void setup_test_1(int* argc, char*** argv)
-{
- *argc = 1;
- *argv = (char**)calloc(sizeof(char*), 1);
- (*argv)[0] = strdup("prog_name");
-}
-
-bool check_test_1(const parsed_opts_t* opts) {
- if ( opts->merged_input_name != NULL
- || opts->unaccounted_header_name != NULL
- || opts->unaccounted_name != NULL
- || strcmp(opts->output_format_string,"%*_%#.%.")
- || opts->verbose == true )
- return false;
- return true;
-}
-
-void setup_test_2(int* argc, char*** argv)
-{
- *argc = 2;
- *argv = (char**)calloc(sizeof(char*), 2);
- (*argv)[0] = strdup("prog_name");
- (*argv)[1] = strdup("merged.bam");
-}
-
-bool check_test_2(const parsed_opts_t* opts) {
- if ( opts->merged_input_name == NULL
- || strcmp(opts->merged_input_name, "merged.bam")
- || opts->unaccounted_header_name != NULL
- || opts->unaccounted_name != NULL
- || strcmp(opts->output_format_string,"%*_%#.%.")
- || opts->verbose == true )
- return false;
- return true;
-}
-
-int samtools_test_parse_args_main(int argc, char**argv)
-{
- // test state
- const int NUM_TESTS = 2;
- int verbose = 0;
- int success = 0;
- int failure = 0;
-
- int getopt_char;
- while ((getopt_char = getopt(argc, argv, "v")) != -1) {
- switch (getopt_char) {
- case 'v':
- ++verbose;
- break;
- default:
- fprintf(samtools_stdout,
- "usage: test_parse_args [-v]\n\n"
- " -v verbose output\n"
- );
- break;
- }
- }
-
- // Setup samtools_stdout and samtools_stderr redirect
- kstring_t res_samtools_stdout = { 0, 0, NULL };
- kstring_t res_samtools_stderr = { 0, 0, NULL };
- FILE* orig_samtools_stdout = fdopen(dup(STDOUT_FILENO), "a"); // Save samtools_stderr
- FILE* orig_samtools_stderr = fdopen(dup(STDERR_FILENO), "a"); // Save samtools_stderr
- char* tempfname_samtools_stdout = (optind < argc)? argv[optind] : "test_parse_args.tmp.o";
- char* tempfname_samtools_stderr = (optind < argc)? argv[optind] : "test_parse_args.tmp.e";
- FILE* check_samtools_stdout = NULL;
- FILE* check_samtools_stderr = NULL;
-
- // Cleanup getopt
- optind = 1;
-
- // setup
- if (verbose) fprintf(orig_samtools_stdout,"BEGIN test 1\n"); // test eliminating a tag that isn't there
- int argc_1;
- char** argv_1;
- setup_test_1(&argc_1, &argv_1);
- if (verbose > 1) {
- fprintf(orig_samtools_stdout, "argc: %d\n", argc_1);
- }
- if (verbose) fprintf(orig_samtools_stdout,"RUN test 1\n");
-
- // test
- xfreopen(tempfname_samtools_stdout, "w", samtools_stdout); // Redirect samtools_stdout to pipe
- xfreopen(tempfname_samtools_stderr, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- parsed_opts_t* result_1 = parse_args(argc_1, argv_1);
- fclose(samtools_stdout);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(orig_samtools_stdout, "END RUN test 1\n");
- if (verbose > 1) {
- fprintf(orig_samtools_stdout, "argc: %d\n", argc_1);
- }
-
- // check result
- res_samtools_stdout.l = res_samtools_stderr.l = 0;
- check_samtools_stdout = fopen(tempfname_samtools_stdout, "r");
- check_samtools_stderr = fopen(tempfname_samtools_stderr, "r");
- if ( !result_1
- && kgetline(&res_samtools_stdout, (kgets_func *)fgets, check_samtools_stdout) >= 0
- && !feof(check_samtools_stdout)
- && res_samtools_stdout.l > 0
- && kgetline(&res_samtools_stderr, (kgets_func *)fgets, check_samtools_stderr) < 0
- && (feof(check_samtools_stderr) || res_samtools_stderr.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(orig_samtools_stdout, "FAIL test 1\n");
- }
- fclose(check_samtools_stderr);
- fclose(check_samtools_stdout);
-
- // teardown
- cleanup_opts(result_1);
- int i = 0;
- for (i = 0; i < argc_1; ++i) {
- free(argv_1[i]);
- }
- free(argv_1);
- if (verbose) fprintf(orig_samtools_stdout, "END test 1\n");
-
- // Cleanup getopt
- optind = 1;
-
- if (verbose) fprintf(orig_samtools_stdout, "BEGIN test 2\n"); // test eliminating a tag that is there
- int argc_2;
- char** argv_2;
- setup_test_2(&argc_2, &argv_2);
- if (verbose > 1) {
- fprintf(orig_samtools_stdout, "argc: %d\n", argc_2);
- }
- if (verbose) fprintf(orig_samtools_stdout, "RUN test 2\n");
-
- // test
- xfreopen(tempfname_samtools_stdout, "w", samtools_stdout); // Redirect samtools_stdout to pipe
- xfreopen(tempfname_samtools_stderr, "w", samtools_stderr); // Redirect samtools_stderr to pipe
- parsed_opts_t* result_2 = parse_args(argc_2, argv_2);
- fclose(samtools_stdout);
- fclose(samtools_stderr);
-
- if (verbose) fprintf(orig_samtools_stdout, "END RUN test 2\n");
- if (verbose > 1) {
- fprintf(orig_samtools_stdout, "argc: %d\n", argc_2);
- }
-
- // check result
- res_samtools_stdout.l = res_samtools_stderr.l = 0;
- check_samtools_stdout = fopen(tempfname_samtools_stdout, "r");
- check_samtools_stderr = fopen(tempfname_samtools_stderr, "r");
- if ( result_2
- && check_test_2(result_2)
- && kgetline(&res_samtools_stdout, (kgets_func *)fgets, check_samtools_stdout) < 0
- && (feof(check_samtools_stdout) || res_samtools_stdout.l == 0)
- && kgetline(&res_samtools_stderr, (kgets_func *)fgets, check_samtools_stderr) < 0
- && (feof(check_samtools_stderr) || res_samtools_stderr.l == 0)) {
- ++success;
- } else {
- ++failure;
- if (verbose) fprintf(orig_samtools_stdout, "FAIL test 2\n");
- }
- fclose(check_samtools_stdout);
- fclose(check_samtools_stderr);
-
- // teardown
- cleanup_opts(result_2);
- int j = 0;
- for (j = 0; j < argc_2; ++j) {
- free(argv_2[j]);
- }
- free(argv_2);
-
- if (verbose) fprintf(orig_samtools_stdout, "END test 2\n");
-
-
- // Cleanup
- free(res_samtools_stdout.s);
- free(res_samtools_stderr.s);
- remove(tempfname_samtools_stdout);
- remove(tempfname_samtools_stderr);
- fclose(orig_samtools_stdout);
- if (failure > 0)
- fprintf(orig_samtools_stderr, "%d failures %d successes\n", failure, success);
- fclose(orig_samtools_stderr);
-
- return (success == NUM_TESTS)? EXIT_SUCCESS : EXIT_FAILURE;
-}
+++ /dev/null
-/* test/test.c -- test harness utility routines.
-
- Copyright (C) 2014, 2016 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <htslib/sam.h>
-
-#include "test.h"
-
-void xfreopen(const char *path, const char *mode, FILE *stream)
-{
- if (freopen(path, mode, stream) == NULL) {
- fprintf(stderr, __FILE__": error reopening %s: %s\n",
- path, strerror(errno));
- exit(2);
- }
-}
-
-void dump_hdr(const bam_hdr_t* hdr)
-{
- printf("n_targets: %d\n", hdr->n_targets);
- printf("ignore_sam_err: %d\n", hdr->ignore_sam_err);
- printf("l_text: %u\n", hdr->l_text);
- printf("idx\ttarget_len\ttarget_name:\n");
- int32_t target;
- for (target = 0; target < hdr->n_targets; ++target) {
- printf("%d\t%u\t\"%s\"\n", target, hdr->target_len[target], hdr->target_name[target]);
- }
- printf("text: \"%s\"\n", hdr->text);
-}
-
-// For tests, just return a constant that can be embedded in expected output.
-const char *samtools_version(void)
-{
- return "x.y.test";
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* test/test.c -- test harness utility routines.
-
- Copyright (C) 2014, 2016 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <htslib/sam.h>
-
-#include "test.h"
-
-void xfreopen(const char *path, const char *mode, FILE *stream)
-{
- if (freopen(path, mode, stream) == NULL) {
- fprintf(samtools_stderr, __FILE__": error reopening %s: %s\n",
- path, strerror(errno));
- exit(2);
- }
-}
-
-void dump_hdr(const bam_hdr_t* hdr)
-{
- fprintf(samtools_stdout, "n_targets: %d\n", hdr->n_targets);
- fprintf(samtools_stdout, "ignore_sam_err: %d\n", hdr->ignore_sam_err);
- fprintf(samtools_stdout, "l_text: %u\n", hdr->l_text);
- fprintf(samtools_stdout, "idx\ttarget_len\ttarget_name:\n");
- int32_t target;
- for (target = 0; target < hdr->n_targets; ++target) {
- fprintf(samtools_stdout, "%d\t%u\t\"%s\"\n", target, hdr->target_len[target], hdr->target_name[target]);
- }
- fprintf(samtools_stdout, "text: \"%s\"\n", hdr->text);
-}
-
-// For tests, just return a constant that can be embedded in expected output.
-const char *samtools_version(void)
-{
- return "x.y.test";
-}
+++ /dev/null
-/* test/test.h -- test harness utility routines.
-
- Copyright (C) 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#ifndef TEST_H
-#define TEST_H
-
-#include <stdio.h>
-#include <htslib/sam.h>
-
-void xfreopen(const char *path, const char *mode, FILE *stream);
-
-void dump_hdr(const bam_hdr_t* hdr);
-
-#endif
+++ /dev/null
-/* test/tview/test_get_rg_sample.c -- tview test cases.
-
- Copyright (C) 2013, 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_tview.c"
-#include <stdbool.h>
-
-const char header_1[] =
-"@HD VN:1.4 SO:undefined\n"
-"@SQ SN:dummy\n"
-"@RG ID:blah SM:po\n";
-
-void setup_test_1(char** header)
-{
- *header = strdup(header_1);
-}
-
-khash_t(kh_rg)* run_test_1(char* header)
-{
- khash_t(kh_rg)* test_result = get_rg_sample(header,"po");
- return test_result;
-}
-
-bool check_test_1(khash_t(kh_rg)* test_result, char* header)
-{
- if (strcmp(header_1, header)) return false;
- // test blah is in there
- if (kh_get(kh_rg, test_result, "blah") == kh_end(test_result))
- {
- return false;
- }
- return true;
-}
-
-void teardown_1(khash_t(kh_rg)* test_result, char* header)
-{
- free(header);
-}
-
-int main(int argc, char** argv)
-{
- const int NUM_TESTS = 1;
- int success = 0;
- int failure = 0;
-
- char* test_header_1;
- setup_test_1(&test_header_1);
- khash_t(kh_rg)* test_result_1 = run_test_1(test_header_1);
- if (!check_test_1(test_result_1, test_header_1))
- failure++;
- else
- success++;
- teardown_1(test_result_1, test_header_1);
-
- if (success == NUM_TESTS) {
- return 0;
- } else {
- fprintf(stderr, "%d failures %d successes\n", failure, success);
- return 1;
- }
-}
+++ /dev/null
-#include "samtools.pysam.h"
-
-/* test/tview/test_get_rg_sample.c -- tview test cases.
-
- Copyright (C) 2013, 2014 Genome Research Ltd.
-
- Author: Martin O. Pollard <mp15@sanger.ac.uk>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE. */
-
-#include <config.h>
-
-#include "../../bam_tview.c"
-#include <stdbool.h>
-
-const char header_1[] =
-"@HD VN:1.4 SO:undefined\n"
-"@SQ SN:dummy\n"
-"@RG ID:blah SM:po\n";
-
-void setup_test_1(char** header)
-{
- *header = strdup(header_1);
-}
-
-khash_t(kh_rg)* run_test_1(char* header)
-{
- khash_t(kh_rg)* test_result = get_rg_sample(header,"po");
- return test_result;
-}
-
-bool check_test_1(khash_t(kh_rg)* test_result, char* header)
-{
- if (strcmp(header_1, header)) return false;
- // test blah is in there
- if (kh_get(kh_rg, test_result, "blah") == kh_end(test_result))
- {
- return false;
- }
- return true;
-}
-
-void teardown_1(khash_t(kh_rg)* test_result, char* header)
-{
- free(header);
-}
-
-int samtools_test_get_rg_sample_main(int argc, char** argv)
-{
- const int NUM_TESTS = 1;
- int success = 0;
- int failure = 0;
-
- char* test_header_1;
- setup_test_1(&test_header_1);
- khash_t(kh_rg)* test_result_1 = run_test_1(test_header_1);
- if (!check_test_1(test_result_1, test_header_1))
- failure++;
- else
- success++;
- teardown_1(test_result_1, test_header_1);
-
- if (success == NUM_TESTS) {
- return 0;
- } else {
- fprintf(samtools_stderr, "%d failures %d successes\n", failure, success);
- return 1;
- }
-}
tmp_file.c - write to and read from a temporary binary file
for fast storage plus added compression.
- Copyright (C) 2017 Genome Research Ltd.
+ Copyright (C) 2017, 2018 Genome Research Ltd.
Author: Andrew Whitwham <aw7@sanger.ac.uk>
tmp->max_data_size = TMP_SAM_MAX_DATA + sizeof(bam1_t); // arbitrary but growable
tmp->ring_buffer_size = TMP_SAM_RING_SIZE; // arbitrary (min 64K) but growable
tmp->comp_buffer_size = LZ4_COMPRESSBOUND(tmp->max_data_size * tmp->group_size);
- tmp->data = NULL;
tmp->ring_buffer = malloc(sizeof(uint8_t) * tmp->ring_buffer_size);
tmp->ring_index = tmp->ring_buffer;
tmp->comp_buffer = malloc(tmp->comp_buffer_size);
/*
- * This does the actual compression and writing to disk. On disk format consists of a
+ * This does the actual compression and writing to a file. The file format consists of a
* single size_t for the size of the compressed data followed by the data itself.
* Returns 0 on success, a negative number on failure.
*/
/*
* Stores an in memory bam structure for writing and if enough are gathered together writes
- * it to disk. Mulitiple alignments compress better that single ones though after a certain number
+ * it to a file. Multiple alignments compress better that single ones though after a certain number
* there is a law of diminishing returns.
* Returns 0 on success, a negative number on failure.
*/
int tmp_file_write(tmp_file_t *tmp, bam1_t *inbam) {
- if ((tmp->input_size + sizeof(bam1_t) + inbam->l_data) >= tmp->ring_buffer_size) {
+ if ((tmp->offset + tmp->input_size + sizeof(bam1_t) + inbam->l_data) >= tmp->ring_buffer_size) {
int ret;
- if ((ret = tmp_file_grow_ring_buffer(tmp, (tmp->input_size + sizeof(bam1_t) + inbam->l_data) * 5))) {
+ if ((ret = tmp_file_grow_ring_buffer(tmp, (tmp->offset + tmp->input_size + sizeof(bam1_t) + inbam->l_data) * 2))) {
tmp_print_error(tmp, "[tmp_file] Error: input line too big. (%ld).\n",
(tmp->input_size + inbam->l_data));
/*
- * Closes the file after writing out any remaining alignments. Adds a size_t 0 to
- * mark the end of the file. Companion function to tmp_file_open_read below.
- * Returns 0 on success, a negative number on failure.
- */
-int tmp_file_close_write(tmp_file_t *tmp) {
- size_t terminator = 0;
-
- if (tmp->entry_number) {
- int ret;
-
- if ((ret = tmp_file_write_to_file(tmp))) {
- return ret;
- }
- }
-
- if (fwrite(&terminator, sizeof(size_t), 1, tmp->fp) < 1) {
- tmp_print_error(tmp, "[tmp_file] Error: tmp file write terminator failed.\n");
- return TMP_SAM_FILE_ERROR;
- }
-
- if (fclose(tmp->fp)) {
- tmp_print_error(tmp, "[tmp_file] Error: closing tmp file %s failed.\n", tmp->name);
- return TMP_SAM_FILE_ERROR;
- }
-
- LZ4_freeStream(tmp->stream);
-
- return TMP_SAM_OK;
-}
-
-
-/*
- * Opens the file for reading. Optionally, if given a pointer to an existing
- * bam1_t structure, it will free the data entry to prevent memory leaks.
- * Companion function to tmp_file_close_write above.
- * Returns 0 on success, a negative number on failure.
- */
-int tmp_file_open_read(tmp_file_t *tmp, bam1_t *inbam) {
-
- if ((tmp->fp = fopen(tmp->name, "rb")) == NULL) {
- tmp_print_error(tmp, "[tmp_file] Error: unable to open read file %s.\n", tmp->name);
- return TMP_SAM_FILE_ERROR;
- }
-
- tmp->dstream = LZ4_createStreamDecode();
- tmp->offset = 0;
-
- if (inbam) {
- free(inbam->data);
- }
-
- if (!tmp->dstream) {
- tmp_print_error(tmp, "[tmp_file] Error: unable to allocate compression stream.\n");
- return TMP_SAM_MEM_ERROR;
- }
-
-
- return TMP_SAM_OK;
-}
-
-
-/*
- * An alternative to tmp_file_close_write that does the same job without actually
- * closing the file. Companion function to tmp_file_begin_read below.
+ * Marks the end of file writing. Adds a size_t 0 to mark the end of
+ * the file. Companion function to tmp_file_begin_read below.
* Returns 0 on success, a negative number on failure.
*/
int tmp_file_end_write(tmp_file_t *tmp) {
/*
- * An alternative to tmp_file_open_read but works on an open file.
+ * Prepares the file for reading.
* Companion function to tmp_file_end_write above.
* Returns 0 on success, a negative number on failure.
*/
-int tmp_file_begin_read(tmp_file_t *tmp, bam1_t *inbam) {
+int tmp_file_begin_read(tmp_file_t *tmp) {
rewind(tmp->fp);
tmp->offset = 0;
tmp->entry_number = tmp->group_size;
- if (inbam) {
- free(inbam->data);
- }
-
if (!tmp->dstream) {
tmp_print_error(tmp, "[tmp_file] Error: unable to allocate compression stream.\n");
return TMP_SAM_MEM_ERROR;
/*
- * Read the next alignment, either from memory or from disk.
+ * Read the next alignment, either from memory or from a file.
* Returns size of entry on success, 0 on end of file or a negative on error.
*/
int tmp_file_read(tmp_file_t *tmp, bam1_t *inbam) {
int entry_size;
+ uint8_t *data = inbam->data;
+
+ /* while tmp_file_read assumes that the same bam1_t variable
+ is being used in each call, this may not be the case. So
+ default to the lowest memory size for safety. */
+ if (tmp->data_size > inbam->m_data) {
+ tmp->data_size = inbam->m_data;
+ }
if (tmp->entry_number == tmp->group_size) {
// read more data
tmp->ring_index = tmp->ring_buffer + tmp->offset;
memcpy(inbam, tmp->ring_index, sizeof(bam1_t));
+ inbam->data = data; // put the pointer to real bam data back
if ((unsigned int)inbam->l_data > tmp->data_size) {
- if ((tmp->data = realloc(tmp->data, sizeof(uint8_t) * inbam->l_data)) == NULL) {
- tmp_print_error(tmp, "[tmp_file] Error: unable to allocate tmp data memory.\n");
+ uint8_t *tmp_data;
+ tmp->data_size = inbam->l_data; kroundup32(tmp->data_size);
+
+ if ((tmp_data = realloc(inbam->data, sizeof(uint8_t) * tmp->data_size)) == NULL) {
+ tmp_print_error(tmp, "[tmp_file] Error: unable to allocate tmp bam data memory.\n");
return TMP_SAM_MEM_ERROR;
}
- tmp->data_size = inbam->l_data;
+ inbam->data = tmp_data;
}
- inbam->data = tmp->data;
+ inbam->m_data = tmp->data_size; // set to the actual data size
+
entry_size = sizeof(bam1_t);
memcpy(inbam->data, tmp->ring_index + entry_size, inbam->l_data);
/*
- * Frees up memory, closes the file and optionally deletes it. Giving this function
- * pointer to the bam1_t structure used for reading will set its data value to null,
- * preventing bam_destroy1() from trying to free already freed memory.
- * Returns 0 on success, a negative number or EOF on failure.
+ * Frees up memory, closes the file and deletes it.
+ * Returns 0 on success or EOF on failure.
*/
-int tmp_file_destroy(tmp_file_t *tmp, bam1_t *inbam, int delete) {
+int tmp_file_destroy(tmp_file_t *tmp) {
int ret = 0;
ret = fclose(tmp->fp);
- if (delete && ret == 0) {
- if (unlink(tmp->name)) {
- tmp_print_error(tmp, "[tmp_file] Error: unable to delete file %s.\n", tmp->name);
- ret = TMP_SAM_FILE_ERROR;
- }
- }
-
LZ4_freeStreamDecode(tmp->dstream);
free(tmp->ring_buffer);
free(tmp->comp_buffer);
free(tmp->name);
- free(tmp->data);
free(tmp->dict);
-
- if (inbam) {
- inbam->data = NULL;
- }
-
return ret;
}
tmp_file.c - write to and read from a temporary binary file
for fast storage plus added compression.
- Copyright (C) 2017 Genome Research Ltd.
+ Copyright (C) 2017, 2018 Genome Research Ltd.
Author: Andrew Whitwham <aw7@sanger.ac.uk>
tmp->max_data_size = TMP_SAM_MAX_DATA + sizeof(bam1_t); // arbitrary but growable
tmp->ring_buffer_size = TMP_SAM_RING_SIZE; // arbitrary (min 64K) but growable
tmp->comp_buffer_size = LZ4_COMPRESSBOUND(tmp->max_data_size * tmp->group_size);
- tmp->data = NULL;
tmp->ring_buffer = malloc(sizeof(uint8_t) * tmp->ring_buffer_size);
tmp->ring_index = tmp->ring_buffer;
tmp->comp_buffer = malloc(tmp->comp_buffer_size);
/*
- * This does the actual compression and writing to disk. On disk format consists of a
+ * This does the actual compression and writing to a file. The file format consists of a
* single size_t for the size of the compressed data followed by the data itself.
* Returns 0 on success, a negative number on failure.
*/
/*
* Stores an in memory bam structure for writing and if enough are gathered together writes
- * it to disk. Mulitiple alignments compress better that single ones though after a certain number
+ * it to a file. Multiple alignments compress better that single ones though after a certain number
* there is a law of diminishing returns.
* Returns 0 on success, a negative number on failure.
*/
int tmp_file_write(tmp_file_t *tmp, bam1_t *inbam) {
- if ((tmp->input_size + sizeof(bam1_t) + inbam->l_data) >= tmp->ring_buffer_size) {
+ if ((tmp->offset + tmp->input_size + sizeof(bam1_t) + inbam->l_data) >= tmp->ring_buffer_size) {
int ret;
- if ((ret = tmp_file_grow_ring_buffer(tmp, (tmp->input_size + sizeof(bam1_t) + inbam->l_data) * 5))) {
+ if ((ret = tmp_file_grow_ring_buffer(tmp, (tmp->offset + tmp->input_size + sizeof(bam1_t) + inbam->l_data) * 2))) {
tmp_print_error(tmp, "[tmp_file] Error: input line too big. (%ld).\n",
(tmp->input_size + inbam->l_data));
/*
- * Closes the file after writing out any remaining alignments. Adds a size_t 0 to
- * mark the end of the file. Companion function to tmp_file_open_read below.
- * Returns 0 on success, a negative number on failure.
- */
-int tmp_file_close_write(tmp_file_t *tmp) {
- size_t terminator = 0;
-
- if (tmp->entry_number) {
- int ret;
-
- if ((ret = tmp_file_write_to_file(tmp))) {
- return ret;
- }
- }
-
- if (fwrite(&terminator, sizeof(size_t), 1, tmp->fp) < 1) {
- tmp_print_error(tmp, "[tmp_file] Error: tmp file write terminator failed.\n");
- return TMP_SAM_FILE_ERROR;
- }
-
- if (fclose(tmp->fp)) {
- tmp_print_error(tmp, "[tmp_file] Error: closing tmp file %s failed.\n", tmp->name);
- return TMP_SAM_FILE_ERROR;
- }
-
- LZ4_freeStream(tmp->stream);
-
- return TMP_SAM_OK;
-}
-
-
-/*
- * Opens the file for reading. Optionally, if given a pointer to an existing
- * bam1_t structure, it will free the data entry to prevent memory leaks.
- * Companion function to tmp_file_close_write above.
- * Returns 0 on success, a negative number on failure.
- */
-int tmp_file_open_read(tmp_file_t *tmp, bam1_t *inbam) {
-
- if ((tmp->fp = fopen(tmp->name, "rb")) == NULL) {
- tmp_print_error(tmp, "[tmp_file] Error: unable to open read file %s.\n", tmp->name);
- return TMP_SAM_FILE_ERROR;
- }
-
- tmp->dstream = LZ4_createStreamDecode();
- tmp->offset = 0;
-
- if (inbam) {
- free(inbam->data);
- }
-
- if (!tmp->dstream) {
- tmp_print_error(tmp, "[tmp_file] Error: unable to allocate compression stream.\n");
- return TMP_SAM_MEM_ERROR;
- }
-
-
- return TMP_SAM_OK;
-}
-
-
-/*
- * An alternative to tmp_file_close_write that does the same job without actually
- * closing the file. Companion function to tmp_file_begin_read below.
+ * Marks the end of file writing. Adds a size_t 0 to mark the end of
+ * the file. Companion function to tmp_file_begin_read below.
* Returns 0 on success, a negative number on failure.
*/
int tmp_file_end_write(tmp_file_t *tmp) {
/*
- * An alternative to tmp_file_open_read but works on an open file.
+ * Prepares the file for reading.
* Companion function to tmp_file_end_write above.
* Returns 0 on success, a negative number on failure.
*/
-int tmp_file_begin_read(tmp_file_t *tmp, bam1_t *inbam) {
+int tmp_file_begin_read(tmp_file_t *tmp) {
rewind(tmp->fp);
tmp->offset = 0;
tmp->entry_number = tmp->group_size;
- if (inbam) {
- free(inbam->data);
- }
-
if (!tmp->dstream) {
tmp_print_error(tmp, "[tmp_file] Error: unable to allocate compression stream.\n");
return TMP_SAM_MEM_ERROR;
/*
- * Read the next alignment, either from memory or from disk.
+ * Read the next alignment, either from memory or from a file.
* Returns size of entry on success, 0 on end of file or a negative on error.
*/
int tmp_file_read(tmp_file_t *tmp, bam1_t *inbam) {
int entry_size;
+ uint8_t *data = inbam->data;
+
+ /* while tmp_file_read assumes that the same bam1_t variable
+ is being used in each call, this may not be the case. So
+ default to the lowest memory size for safety. */
+ if (tmp->data_size > inbam->m_data) {
+ tmp->data_size = inbam->m_data;
+ }
if (tmp->entry_number == tmp->group_size) {
// read more data
tmp->ring_index = tmp->ring_buffer + tmp->offset;
memcpy(inbam, tmp->ring_index, sizeof(bam1_t));
+ inbam->data = data; // put the pointer to real bam data back
if ((unsigned int)inbam->l_data > tmp->data_size) {
- if ((tmp->data = realloc(tmp->data, sizeof(uint8_t) * inbam->l_data)) == NULL) {
- tmp_print_error(tmp, "[tmp_file] Error: unable to allocate tmp data memory.\n");
+ uint8_t *tmp_data;
+ tmp->data_size = inbam->l_data; kroundup32(tmp->data_size);
+
+ if ((tmp_data = realloc(inbam->data, sizeof(uint8_t) * tmp->data_size)) == NULL) {
+ tmp_print_error(tmp, "[tmp_file] Error: unable to allocate tmp bam data memory.\n");
return TMP_SAM_MEM_ERROR;
}
- tmp->data_size = inbam->l_data;
+ inbam->data = tmp_data;
}
- inbam->data = tmp->data;
+ inbam->m_data = tmp->data_size; // set to the actual data size
+
entry_size = sizeof(bam1_t);
memcpy(inbam->data, tmp->ring_index + entry_size, inbam->l_data);
/*
- * Frees up memory, closes the file and optionally deletes it. Giving this function
- * pointer to the bam1_t structure used for reading will set its data value to null,
- * preventing bam_destroy1() from trying to free already freed memory.
- * Returns 0 on success, a negative number or EOF on failure.
+ * Frees up memory, closes the file and deletes it.
+ * Returns 0 on success or EOF on failure.
*/
-int tmp_file_destroy(tmp_file_t *tmp, bam1_t *inbam, int delete) {
+int tmp_file_destroy(tmp_file_t *tmp) {
int ret = 0;
ret = fclose(tmp->fp);
- if (delete && ret == 0) {
- if (unlink(tmp->name)) {
- tmp_print_error(tmp, "[tmp_file] Error: unable to delete file %s.\n", tmp->name);
- ret = TMP_SAM_FILE_ERROR;
- }
- }
-
LZ4_freeStreamDecode(tmp->dstream);
free(tmp->ring_buffer);
free(tmp->comp_buffer);
free(tmp->name);
- free(tmp->data);
free(tmp->dict);
-
- if (inbam) {
- inbam->data = NULL;
- }
-
return ret;
}
tmp_file.h - write to and read from a temporary binary file
for fast storage plus added compression.
- Copyright (C) 2017 Genome Research Ltd.
+ Copyright (C) 2017, 2018 Genome Research Ltd.
Author: Andrew Whitwham <aw7@sanger.ac.uk>
size_t ring_buffer_size;
size_t comp_buffer_size;
size_t offset;
- uint8_t *data;
uint8_t *ring_buffer;
uint8_t *ring_index;
char *comp_buffer;
/*
* Stores an in memory bam structure for writing and if enough are gathered together writes
- * it to disk. Mulitiple alignments compress better that single ones though after a certain number
+ * it to a file. Multiple alignments compress better that single ones though after a certain number
* there is a law of diminishing returns.
* Returns 0 on success, a negative number on failure.
*/
/*
- * Closes the file after writing out any remaining alignments. Adds a size_t 0 to
- * mark the end of the file. Companion function to tmp_file_open_read below.
- * Returns 0 on success, a negative number on failure.
- */
-int tmp_file_close_write(tmp_file_t *tmp);
-
-
-/*
- * Opens the file for reading. Optionally, if given a pointer to an existing
- * bam1_t structure, it will free the data entry to prevent memory leaks.
- * Companion function to tmp_file_close_write above.
- * Returns 0 on success, a negative number on failure.
- */
-int tmp_file_open_read(tmp_file_t *tmp, bam1_t *inbam);
-
-
-/*
- * An alternative to tmp_file_close_write that does the same job without actually
- * closing the file. Companion function to tmp_file_begin_read below.
+ * Marks the end of file writing. Adds a size_t 0 to mark the end of
+ * the file. Companion function to tmp_file_begin_read below.
* Returns 0 on success, a negative number on failure.
*/
int tmp_file_end_write(tmp_file_t *tmp);
/*
- * An alternative to tmp_file_open_read but works on an open file.
+ * Prepares the file for reading.
* Companion function to tmp_file_end_write above.
* Returns 0 on success, a negative number on failure.
*/
-int tmp_file_begin_read(tmp_file_t *tmp, bam1_t *inbam);
+int tmp_file_begin_read(tmp_file_t *tmp);
/*
- * Read the next alignment, either from memory or from disk.
+ * Read the next alignment, either from memory or from a file.
* Returns size of entry on success, 0 on end of file or a negative on error.
*/
int tmp_file_read(tmp_file_t *tmp, bam1_t *inbam);
/*
- * Frees up memory, closes the file and optionally deletes it. Giving this function
- * pointer to the bam1_t structure used for reading will set its data value to null,
- * preventing bam_destroy1() from trying to free already freed memory.
- * Returns 0 on success, a negative number or EOF on failure.
+ * Frees up memory, closes the file and deletes it.
+ * Returns 0 on success or EOF on failure.
*/
-int tmp_file_destroy(tmp_file_t *tmp, bam1_t *inbam, int delete);
+int tmp_file_destroy(tmp_file_t *tmp);
#ifdef __cplusplus
}
+++ /dev/null
-#define SAMTOOLS_VERSION "1.9"
--- /dev/null
+#!/bin/sh
+# version.sh -- Script to build the htslib version string
+#
+# Author : James Bonfield <jkb@sanger.ac.uk>
+#
+# Copyright (C) 2017, 2019 Genome Research Ltd.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# Master version, for use in tarballs or non-git source copies
+VERSION=1.10
+
+# If we have a git clone, then check against the current tag
+if [ -e .git ]
+then
+ # If we ever get to 10.x this will need to be more liberal
+ VERSION=`git describe --match '[0-9].[0-9]*' --dirty --always`
+fi
+
+echo $VERSION
+++ /dev/null
-/* Public Domain Curses */
-
-/* $Id: curses.h,v 1.295 2008/07/15 17:13:25 wmcbrine Exp $ */
-
-/*----------------------------------------------------------------------*
- * PDCurses *
- *----------------------------------------------------------------------*/
-
-#ifndef __PDCURSES__
-#define __PDCURSES__ 1
-
-/*man-start**************************************************************
-
-PDCurses definitions list: (Only define those needed)
-
- XCURSES True if compiling for X11.
- PDC_RGB True if you want to use RGB color definitions
- (Red = 1, Green = 2, Blue = 4) instead of BGR.
- PDC_WIDE True if building wide-character support.
- PDC_DLL_BUILD True if building a Win32 DLL.
- NCURSES_MOUSE_VERSION Use the ncurses mouse API instead
- of PDCurses' traditional mouse API.
-
-PDCurses portable platform definitions list:
-
- PDC_BUILD Defines API build version.
- PDCURSES Enables access to PDCurses-only routines.
- XOPEN Always true.
- SYSVcurses True if you are compiling for SYSV portability.
- BSDcurses True if you are compiling for BSD portability.
-
-**man-end****************************************************************/
-
-#define PDC_BUILD 3401
-#define PDCURSES 1 /* PDCurses-only routines */
-#define XOPEN 1 /* X/Open Curses routines */
-#define SYSVcurses 1 /* System V Curses routines */
-#define BSDcurses 1 /* BSD Curses routines */
-#define CHTYPE_LONG 1 /* size of chtype; long */
-
-/*----------------------------------------------------------------------*/
-
-#include <stdarg.h>
-#include <stddef.h>
-#include <stdio.h> /* Required by X/Open usage below */
-
-#ifdef PDC_WIDE
-# include <wchar.h>
-#endif
-
-#if defined(__cplusplus) || defined(__cplusplus__) || defined(__CPLUSPLUS)
-extern "C"
-{
-# define bool _bool
-#endif
-
-/*----------------------------------------------------------------------
- *
- * PDCurses Manifest Constants
- *
- */
-
-#ifndef FALSE
-# define FALSE 0
-#endif
-#ifndef TRUE
-# define TRUE 1
-#endif
-#ifndef NULL
-# define NULL (void *)0
-#endif
-#ifndef ERR
-# define ERR (-1)
-#endif
-#ifndef OK
-# define OK 0
-#endif
-
-/*----------------------------------------------------------------------
- *
- * PDCurses Type Declarations
- *
- */
-
-typedef unsigned char bool; /* PDCurses Boolean type */
-
-#ifdef CHTYPE_LONG
-# if _LP64
-typedef unsigned int chtype;
-# else
-typedef unsigned long chtype; /* 16-bit attr + 16-bit char */
-# endif
-#else
-typedef unsigned short chtype; /* 8-bit attr + 8-bit char */
-#endif
-
-#ifdef PDC_WIDE
-typedef chtype cchar_t;
-#endif
-
-typedef chtype attr_t;
-
-/*----------------------------------------------------------------------
- *
- * PDCurses Mouse Interface -- SYSVR4, with extensions
- *
- */
-
-typedef struct
-{
- int x; /* absolute column, 0 based, measured in characters */
- int y; /* absolute row, 0 based, measured in characters */
- short button[3]; /* state of each button */
- int changes; /* flags indicating what has changed with the mouse */
-} MOUSE_STATUS;
-
-#define BUTTON_RELEASED 0x0000
-#define BUTTON_PRESSED 0x0001
-#define BUTTON_CLICKED 0x0002
-#define BUTTON_DOUBLE_CLICKED 0x0003
-#define BUTTON_TRIPLE_CLICKED 0x0004
-#define BUTTON_MOVED 0x0005 /* PDCurses */
-#define WHEEL_SCROLLED 0x0006 /* PDCurses */
-#define BUTTON_ACTION_MASK 0x0007 /* PDCurses */
-
-#define PDC_BUTTON_SHIFT 0x0008 /* PDCurses */
-#define PDC_BUTTON_CONTROL 0x0010 /* PDCurses */
-#define PDC_BUTTON_ALT 0x0020 /* PDCurses */
-#define BUTTON_MODIFIER_MASK 0x0038 /* PDCurses */
-
-#define MOUSE_X_POS (Mouse_status.x)
-#define MOUSE_Y_POS (Mouse_status.y)
-
-/*
- * Bits associated with the .changes field:
- * 3 2 1 0
- * 210987654321098765432109876543210
- * 1 <- button 1 has changed
- * 10 <- button 2 has changed
- * 100 <- button 3 has changed
- * 1000 <- mouse has moved
- * 10000 <- mouse position report
- * 100000 <- mouse wheel up
- * 1000000 <- mouse wheel down
- */
-
-#define PDC_MOUSE_MOVED 0x0008
-#define PDC_MOUSE_POSITION 0x0010
-#define PDC_MOUSE_WHEEL_UP 0x0020
-#define PDC_MOUSE_WHEEL_DOWN 0x0040
-
-#define A_BUTTON_CHANGED (Mouse_status.changes & 7)
-#define MOUSE_MOVED (Mouse_status.changes & PDC_MOUSE_MOVED)
-#define MOUSE_POS_REPORT (Mouse_status.changes & PDC_MOUSE_POSITION)
-#define BUTTON_CHANGED(x) (Mouse_status.changes & (1 << ((x) - 1)))
-#define BUTTON_STATUS(x) (Mouse_status.button[(x) - 1])
-#define MOUSE_WHEEL_UP (Mouse_status.changes & PDC_MOUSE_WHEEL_UP)
-#define MOUSE_WHEEL_DOWN (Mouse_status.changes & PDC_MOUSE_WHEEL_DOWN)
-
-/* mouse bit-masks */
-
-#define BUTTON1_RELEASED 0x00000001L
-#define BUTTON1_PRESSED 0x00000002L
-#define BUTTON1_CLICKED 0x00000004L
-#define BUTTON1_DOUBLE_CLICKED 0x00000008L
-#define BUTTON1_TRIPLE_CLICKED 0x00000010L
-#define BUTTON1_MOVED 0x00000010L /* PDCurses */
-
-#define BUTTON2_RELEASED 0x00000020L
-#define BUTTON2_PRESSED 0x00000040L
-#define BUTTON2_CLICKED 0x00000080L
-#define BUTTON2_DOUBLE_CLICKED 0x00000100L
-#define BUTTON2_TRIPLE_CLICKED 0x00000200L
-#define BUTTON2_MOVED 0x00000200L /* PDCurses */
-
-#define BUTTON3_RELEASED 0x00000400L
-#define BUTTON3_PRESSED 0x00000800L
-#define BUTTON3_CLICKED 0x00001000L
-#define BUTTON3_DOUBLE_CLICKED 0x00002000L
-#define BUTTON3_TRIPLE_CLICKED 0x00004000L
-#define BUTTON3_MOVED 0x00004000L /* PDCurses */
-
-/* For the ncurses-compatible functions only, BUTTON4_PRESSED and
- BUTTON5_PRESSED are returned for mouse scroll wheel up and down;
- otherwise PDCurses doesn't support buttons 4 and 5 */
-
-#define BUTTON4_RELEASED 0x00008000L
-#define BUTTON4_PRESSED 0x00010000L
-#define BUTTON4_CLICKED 0x00020000L
-#define BUTTON4_DOUBLE_CLICKED 0x00040000L
-#define BUTTON4_TRIPLE_CLICKED 0x00080000L
-
-#define BUTTON5_RELEASED 0x00100000L
-#define BUTTON5_PRESSED 0x00200000L
-#define BUTTON5_CLICKED 0x00400000L
-#define BUTTON5_DOUBLE_CLICKED 0x00800000L
-#define BUTTON5_TRIPLE_CLICKED 0x01000000L
-
-#define MOUSE_WHEEL_SCROLL 0x02000000L /* PDCurses */
-#define BUTTON_MODIFIER_SHIFT 0x04000000L /* PDCurses */
-#define BUTTON_MODIFIER_CONTROL 0x08000000L /* PDCurses */
-#define BUTTON_MODIFIER_ALT 0x10000000L /* PDCurses */
-
-#define ALL_MOUSE_EVENTS 0x1fffffffL
-#define REPORT_MOUSE_POSITION 0x20000000L
-
-/* ncurses mouse interface */
-
-typedef unsigned long mmask_t;
-
-typedef struct
-{
- short id; /* unused, always 0 */
- int x, y, z; /* x, y same as MOUSE_STATUS; z unused */
- mmask_t bstate; /* equivalent to changes + button[], but
- in the same format as used for mousemask() */
-} MEVENT;
-
-#ifdef NCURSES_MOUSE_VERSION
-# define BUTTON_SHIFT BUTTON_MODIFIER_SHIFT
-# define BUTTON_CONTROL BUTTON_MODIFIER_CONTROL
-# define BUTTON_CTRL BUTTON_MODIFIER_CONTROL
-# define BUTTON_ALT BUTTON_MODIFIER_ALT
-#else
-# define BUTTON_SHIFT PDC_BUTTON_SHIFT
-# define BUTTON_CONTROL PDC_BUTTON_CONTROL
-# define BUTTON_ALT PDC_BUTTON_ALT
-#endif
-
-/*----------------------------------------------------------------------
- *
- * PDCurses Structure Definitions
- *
- */
-
-typedef struct _win /* definition of a window */
-{
- int _cury; /* current pseudo-cursor */
- int _curx;
- int _maxy; /* max window coordinates */
- int _maxx;
- int _begy; /* origin on screen */
- int _begx;
- int _flags; /* window properties */
- chtype _attrs; /* standard attributes and colors */
- chtype _bkgd; /* background, normally blank */
- bool _clear; /* causes clear at next refresh */
- bool _leaveit; /* leaves cursor where it is */
- bool _scroll; /* allows window scrolling */
- bool _nodelay; /* input character wait flag */
- bool _immed; /* immediate update flag */
- bool _sync; /* synchronise window ancestors */
- bool _use_keypad; /* flags keypad key mode active */
- chtype **_y; /* pointer to line pointer array */
- int *_firstch; /* first changed character in line */
- int *_lastch; /* last changed character in line */
- int _tmarg; /* top of scrolling region */
- int _bmarg; /* bottom of scrolling region */
- int _delayms; /* milliseconds of delay for getch() */
- int _parx, _pary; /* coords relative to parent (0,0) */
- struct _win *_parent; /* subwin's pointer to parent win */
-} WINDOW;
-
-/* Avoid using the SCREEN struct directly -- use the corresponding
- functions if possible. This struct may eventually be made private. */
-
-typedef struct
-{
- bool alive; /* if initscr() called, and not endwin() */
- bool autocr; /* if cr -> lf */
- bool cbreak; /* if terminal unbuffered */
- bool echo; /* if terminal echo */
- bool raw_inp; /* raw input mode (v. cooked input) */
- bool raw_out; /* raw output mode (7 v. 8 bits) */
- bool audible; /* FALSE if the bell is visual */
- bool mono; /* TRUE if current screen is mono */
- bool resized; /* TRUE if TERM has been resized */
- bool orig_attr; /* TRUE if we have the original colors */
- short orig_fore; /* original screen foreground color */
- short orig_back; /* original screen foreground color */
- int cursrow; /* position of physical cursor */
- int curscol; /* position of physical cursor */
- int visibility; /* visibility of cursor */
- int orig_cursor; /* original cursor size */
- int lines; /* new value for LINES */
- int cols; /* new value for COLS */
- unsigned long _trap_mbe; /* trap these mouse button events */
- unsigned long _map_mbe_to_key; /* map mouse buttons to slk */
- int mouse_wait; /* time to wait (in ms) for a
- button release after a press, in
- order to count it as a click */
- int slklines; /* lines in use by slk_init() */
- WINDOW *slk_winptr; /* window for slk */
- int linesrippedoff; /* lines ripped off via ripoffline() */
- int linesrippedoffontop; /* lines ripped off on
- top via ripoffline() */
- int delaytenths; /* 1/10ths second to wait block
- getch() for */
- bool _preserve; /* TRUE if screen background
- to be preserved */
- int _restore; /* specifies if screen background
- to be restored, and how */
- bool save_key_modifiers; /* TRUE if each key modifiers saved
- with each key press */
- bool return_key_modifiers; /* TRUE if modifier keys are
- returned as "real" keys */
- bool key_code; /* TRUE if last key is a special key;
- used internally by get_wch() */
-#ifdef XCURSES
- int XcurscrSize; /* size of Xcurscr shared memory block */
- bool sb_on;
- int sb_viewport_y;
- int sb_viewport_x;
- int sb_total_y;
- int sb_total_x;
- int sb_cur_y;
- int sb_cur_x;
-#endif
- short line_color; /* color of line attributes - default -1 */
-} SCREEN;
-
-/*----------------------------------------------------------------------
- *
- * PDCurses External Variables
- *
- */
-
-#ifdef PDC_DLL_BUILD
-# ifdef CURSES_LIBRARY
-# define PDCEX __declspec(dllexport) extern
-# else
-# define PDCEX __declspec(dllimport)
-# endif
-#else
-# define PDCEX extern
-#endif
-
-PDCEX int LINES; /* terminal height */
-PDCEX int COLS; /* terminal width */
-PDCEX WINDOW *stdscr; /* the default screen window */
-PDCEX WINDOW *curscr; /* the current screen image */
-PDCEX SCREEN *SP; /* curses variables */
-PDCEX MOUSE_STATUS Mouse_status;
-PDCEX int COLORS;
-PDCEX int COLOR_PAIRS;
-PDCEX int TABSIZE;
-PDCEX chtype acs_map[]; /* alternate character set map */
-PDCEX char ttytype[]; /* terminal name/description */
-
-/*man-start**************************************************************
-
-PDCurses Text Attributes
-========================
-
-Originally, PDCurses used a short (16 bits) for its chtype. To include
-color, a number of things had to be sacrificed from the strict Unix and
-System V support. The main problem was fitting all character attributes
-and color into an unsigned char (all 8 bits!).
-
-Today, PDCurses by default uses a long (32 bits) for its chtype, as in
-System V. The short chtype is still available, by undefining CHTYPE_LONG
-and rebuilding the library.
-
-The following is the structure of a win->_attrs chtype:
-
-short form:
-
--------------------------------------------------
-|15|14|13|12|11|10| 9| 8| 7| 6| 5| 4| 3| 2| 1| 0|
--------------------------------------------------
- color number | attrs | character eg 'a'
-
-The available non-color attributes are bold, reverse and blink. Others
-have no effect. The high order char is an index into an array of
-physical colors (defined in color.c) -- 32 foreground/background color
-pairs (5 bits) plus 3 bits for other attributes.
-
-long form:
-
-----------------------------------------------------------------------------
-|31|30|29|28|27|26|25|24|23|22|21|20|19|18|17|16|15|14|13|12|..| 3| 2| 1| 0|
-----------------------------------------------------------------------------
- color number | modifiers | character eg 'a'
-
-The available non-color attributes are bold, underline, invisible,
-right-line, left-line, protect, reverse and blink. 256 color pairs (8
-bits), 8 bits for other attributes, and 16 bits for character data.
-
-**man-end****************************************************************/
-
-/*** Video attribute macros ***/
-
-#define A_NORMAL (chtype)0
-
-#ifdef CHTYPE_LONG
-# define A_ALTCHARSET (chtype)0x00010000
-# define A_RIGHTLINE (chtype)0x00020000
-# define A_LEFTLINE (chtype)0x00040000
-# define A_INVIS (chtype)0x00080000
-# define A_UNDERLINE (chtype)0x00100000
-# define A_REVERSE (chtype)0x00200000
-# define A_BLINK (chtype)0x00400000
-# define A_BOLD (chtype)0x00800000
-
-# define A_ATTRIBUTES (chtype)0xffff0000
-# define A_CHARTEXT (chtype)0x0000ffff
-# define A_COLOR (chtype)0xff000000
-
-# define A_ITALIC A_INVIS
-# define A_PROTECT (A_UNDERLINE | A_LEFTLINE | A_RIGHTLINE)
-
-# define PDC_ATTR_SHIFT 19
-# define PDC_COLOR_SHIFT 24
-#else
-# define A_BOLD (chtype)0x0100 /* X/Open */
-# define A_REVERSE (chtype)0x0200 /* X/Open */
-# define A_BLINK (chtype)0x0400 /* X/Open */
-
-# define A_ATTRIBUTES (chtype)0xff00 /* X/Open */
-# define A_CHARTEXT (chtype)0x00ff /* X/Open */
-# define A_COLOR (chtype)0xf800 /* System V */
-
-# define A_ALTCHARSET A_NORMAL /* X/Open */
-# define A_PROTECT A_NORMAL /* X/Open */
-# define A_UNDERLINE A_NORMAL /* X/Open */
-
-# define A_LEFTLINE A_NORMAL
-# define A_RIGHTLINE A_NORMAL
-# define A_ITALIC A_NORMAL
-# define A_INVIS A_NORMAL
-
-# define PDC_ATTR_SHIFT 8
-# define PDC_COLOR_SHIFT 11
-#endif
-
-#define A_STANDOUT (A_REVERSE | A_BOLD) /* X/Open */
-#define A_DIM A_NORMAL
-
-#define CHR_MSK A_CHARTEXT /* Obsolete */
-#define ATR_MSK A_ATTRIBUTES /* Obsolete */
-#define ATR_NRM A_NORMAL /* Obsolete */
-
-/* For use with attr_t -- X/Open says, "these shall be distinct", so
- this is a non-conforming implementation. */
-
-#define WA_ALTCHARSET A_ALTCHARSET
-#define WA_BLINK A_BLINK
-#define WA_BOLD A_BOLD
-#define WA_DIM A_DIM
-#define WA_INVIS A_INVIS
-#define WA_LEFT A_LEFTLINE
-#define WA_PROTECT A_PROTECT
-#define WA_REVERSE A_REVERSE
-#define WA_RIGHT A_RIGHTLINE
-#define WA_STANDOUT A_STANDOUT
-#define WA_UNDERLINE A_UNDERLINE
-
-#define WA_HORIZONTAL A_NORMAL
-#define WA_LOW A_NORMAL
-#define WA_TOP A_NORMAL
-#define WA_VERTICAL A_NORMAL
-
-/*** Alternate character set macros ***/
-
-/* 'w' = 32-bit chtype; acs_map[] index | A_ALTCHARSET
- 'n' = 16-bit chtype; it gets the fallback set because no bit is
- available for A_ALTCHARSET */
-
-#ifdef CHTYPE_LONG
-# define ACS_PICK(w, n) ((chtype)w | A_ALTCHARSET)
-#else
-# define ACS_PICK(w, n) ((chtype)n)
-#endif
-
-/* VT100-compatible symbols -- box chars */
-
-#define ACS_ULCORNER ACS_PICK('l', '+')
-#define ACS_LLCORNER ACS_PICK('m', '+')
-#define ACS_URCORNER ACS_PICK('k', '+')
-#define ACS_LRCORNER ACS_PICK('j', '+')
-#define ACS_RTEE ACS_PICK('u', '+')
-#define ACS_LTEE ACS_PICK('t', '+')
-#define ACS_BTEE ACS_PICK('v', '+')
-#define ACS_TTEE ACS_PICK('w', '+')
-#define ACS_HLINE ACS_PICK('q', '-')
-#define ACS_VLINE ACS_PICK('x', '|')
-#define ACS_PLUS ACS_PICK('n', '+')
-
-/* VT100-compatible symbols -- other */
-
-#define ACS_S1 ACS_PICK('o', '-')
-#define ACS_S9 ACS_PICK('s', '_')
-#define ACS_DIAMOND ACS_PICK('`', '+')
-#define ACS_CKBOARD ACS_PICK('a', ':')
-#define ACS_DEGREE ACS_PICK('f', '\'')
-#define ACS_PLMINUS ACS_PICK('g', '#')
-#define ACS_BULLET ACS_PICK('~', 'o')
-
-/* Teletype 5410v1 symbols -- these are defined in SysV curses, but
- are not well-supported by most terminals. Stick to VT100 characters
- for optimum portability. */
-
-#define ACS_LARROW ACS_PICK(',', '<')
-#define ACS_RARROW ACS_PICK('+', '>')
-#define ACS_DARROW ACS_PICK('.', 'v')
-#define ACS_UARROW ACS_PICK('-', '^')
-#define ACS_BOARD ACS_PICK('h', '#')
-#define ACS_LANTERN ACS_PICK('i', '*')
-#define ACS_BLOCK ACS_PICK('0', '#')
-
-/* That goes double for these -- undocumented SysV symbols. Don't use
- them. */
-
-#define ACS_S3 ACS_PICK('p', '-')
-#define ACS_S7 ACS_PICK('r', '-')
-#define ACS_LEQUAL ACS_PICK('y', '<')
-#define ACS_GEQUAL ACS_PICK('z', '>')
-#define ACS_PI ACS_PICK('{', 'n')
-#define ACS_NEQUAL ACS_PICK('|', '+')
-#define ACS_STERLING ACS_PICK('}', 'L')
-
-/* Box char aliases */
-
-#define ACS_BSSB ACS_ULCORNER
-#define ACS_SSBB ACS_LLCORNER
-#define ACS_BBSS ACS_URCORNER
-#define ACS_SBBS ACS_LRCORNER
-#define ACS_SBSS ACS_RTEE
-#define ACS_SSSB ACS_LTEE
-#define ACS_SSBS ACS_BTEE
-#define ACS_BSSS ACS_TTEE
-#define ACS_BSBS ACS_HLINE
-#define ACS_SBSB ACS_VLINE
-#define ACS_SSSS ACS_PLUS
-
-/* cchar_t aliases */
-
-#ifdef PDC_WIDE
-# define WACS_ULCORNER (&(acs_map['l']))
-# define WACS_LLCORNER (&(acs_map['m']))
-# define WACS_URCORNER (&(acs_map['k']))
-# define WACS_LRCORNER (&(acs_map['j']))
-# define WACS_RTEE (&(acs_map['u']))
-# define WACS_LTEE (&(acs_map['t']))
-# define WACS_BTEE (&(acs_map['v']))
-# define WACS_TTEE (&(acs_map['w']))
-# define WACS_HLINE (&(acs_map['q']))
-# define WACS_VLINE (&(acs_map['x']))
-# define WACS_PLUS (&(acs_map['n']))
-
-# define WACS_S1 (&(acs_map['o']))
-# define WACS_S9 (&(acs_map['s']))
-# define WACS_DIAMOND (&(acs_map['`']))
-# define WACS_CKBOARD (&(acs_map['a']))
-# define WACS_DEGREE (&(acs_map['f']))
-# define WACS_PLMINUS (&(acs_map['g']))
-# define WACS_BULLET (&(acs_map['~']))
-
-# define WACS_LARROW (&(acs_map[',']))
-# define WACS_RARROW (&(acs_map['+']))
-# define WACS_DARROW (&(acs_map['.']))
-# define WACS_UARROW (&(acs_map['-']))
-# define WACS_BOARD (&(acs_map['h']))
-# define WACS_LANTERN (&(acs_map['i']))
-# define WACS_BLOCK (&(acs_map['0']))
-
-# define WACS_S3 (&(acs_map['p']))
-# define WACS_S7 (&(acs_map['r']))
-# define WACS_LEQUAL (&(acs_map['y']))
-# define WACS_GEQUAL (&(acs_map['z']))
-# define WACS_PI (&(acs_map['{']))
-# define WACS_NEQUAL (&(acs_map['|']))
-# define WACS_STERLING (&(acs_map['}']))
-
-# define WACS_BSSB WACS_ULCORNER
-# define WACS_SSBB WACS_LLCORNER
-# define WACS_BBSS WACS_URCORNER
-# define WACS_SBBS WACS_LRCORNER
-# define WACS_SBSS WACS_RTEE
-# define WACS_SSSB WACS_LTEE
-# define WACS_SSBS WACS_BTEE
-# define WACS_BSSS WACS_TTEE
-# define WACS_BSBS WACS_HLINE
-# define WACS_SBSB WACS_VLINE
-# define WACS_SSSS WACS_PLUS
-#endif
-
-/*** Color macros ***/
-
-#define COLOR_BLACK 0
-
-#ifdef PDC_RGB /* RGB */
-# define COLOR_RED 1
-# define COLOR_GREEN 2
-# define COLOR_BLUE 4
-#else /* BGR */
-# define COLOR_BLUE 1
-# define COLOR_GREEN 2
-# define COLOR_RED 4
-#endif
-
-#define COLOR_CYAN (COLOR_BLUE | COLOR_GREEN)
-#define COLOR_MAGENTA (COLOR_RED | COLOR_BLUE)
-#define COLOR_YELLOW (COLOR_RED | COLOR_GREEN)
-
-#define COLOR_WHITE 7
-
-/*----------------------------------------------------------------------
- *
- * Function and Keypad Key Definitions.
- * Many are just for compatibility.
- *
- */
-
-#define KEY_CODE_YES 0x100 /* If get_wch() gives a key code */
-
-#define KEY_BREAK 0x101 /* Not on PC KBD */
-#define KEY_DOWN 0x102 /* Down arrow key */
-#define KEY_UP 0x103 /* Up arrow key */
-#define KEY_LEFT 0x104 /* Left arrow key */
-#define KEY_RIGHT 0x105 /* Right arrow key */
-#define KEY_HOME 0x106 /* home key */
-#define KEY_BACKSPACE 0x107 /* not on pc */
-#define KEY_F0 0x108 /* function keys; 64 reserved */
-
-#define KEY_DL 0x148 /* delete line */
-#define KEY_IL 0x149 /* insert line */
-#define KEY_DC 0x14a /* delete character */
-#define KEY_IC 0x14b /* insert char or enter ins mode */
-#define KEY_EIC 0x14c /* exit insert char mode */
-#define KEY_CLEAR 0x14d /* clear screen */
-#define KEY_EOS 0x14e /* clear to end of screen */
-#define KEY_EOL 0x14f /* clear to end of line */
-#define KEY_SF 0x150 /* scroll 1 line forward */
-#define KEY_SR 0x151 /* scroll 1 line back (reverse) */
-#define KEY_NPAGE 0x152 /* next page */
-#define KEY_PPAGE 0x153 /* previous page */
-#define KEY_STAB 0x154 /* set tab */
-#define KEY_CTAB 0x155 /* clear tab */
-#define KEY_CATAB 0x156 /* clear all tabs */
-#define KEY_ENTER 0x157 /* enter or send (unreliable) */
-#define KEY_SRESET 0x158 /* soft/reset (partial/unreliable) */
-#define KEY_RESET 0x159 /* reset/hard reset (unreliable) */
-#define KEY_PRINT 0x15a /* print/copy */
-#define KEY_LL 0x15b /* home down/bottom (lower left) */
-#define KEY_ABORT 0x15c /* abort/terminate key (any) */
-#define KEY_SHELP 0x15d /* short help */
-#define KEY_LHELP 0x15e /* long help */
-#define KEY_BTAB 0x15f /* Back tab key */
-#define KEY_BEG 0x160 /* beg(inning) key */
-#define KEY_CANCEL 0x161 /* cancel key */
-#define KEY_CLOSE 0x162 /* close key */
-#define KEY_COMMAND 0x163 /* cmd (command) key */
-#define KEY_COPY 0x164 /* copy key */
-#define KEY_CREATE 0x165 /* create key */
-#define KEY_END 0x166 /* end key */
-#define KEY_EXIT 0x167 /* exit key */
-#define KEY_FIND 0x168 /* find key */
-#define KEY_HELP 0x169 /* help key */
-#define KEY_MARK 0x16a /* mark key */
-#define KEY_MESSAGE 0x16b /* message key */
-#define KEY_MOVE 0x16c /* move key */
-#define KEY_NEXT 0x16d /* next object key */
-#define KEY_OPEN 0x16e /* open key */
-#define KEY_OPTIONS 0x16f /* options key */
-#define KEY_PREVIOUS 0x170 /* previous object key */
-#define KEY_REDO 0x171 /* redo key */
-#define KEY_REFERENCE 0x172 /* ref(erence) key */
-#define KEY_REFRESH 0x173 /* refresh key */
-#define KEY_REPLACE 0x174 /* replace key */
-#define KEY_RESTART 0x175 /* restart key */
-#define KEY_RESUME 0x176 /* resume key */
-#define KEY_SAVE 0x177 /* save key */
-#define KEY_SBEG 0x178 /* shifted beginning key */
-#define KEY_SCANCEL 0x179 /* shifted cancel key */
-#define KEY_SCOMMAND 0x17a /* shifted command key */
-#define KEY_SCOPY 0x17b /* shifted copy key */
-#define KEY_SCREATE 0x17c /* shifted create key */
-#define KEY_SDC 0x17d /* shifted delete char key */
-#define KEY_SDL 0x17e /* shifted delete line key */
-#define KEY_SELECT 0x17f /* select key */
-#define KEY_SEND 0x180 /* shifted end key */
-#define KEY_SEOL 0x181 /* shifted clear line key */
-#define KEY_SEXIT 0x182 /* shifted exit key */
-#define KEY_SFIND 0x183 /* shifted find key */
-#define KEY_SHOME 0x184 /* shifted home key */
-#define KEY_SIC 0x185 /* shifted input key */
-
-#define KEY_SLEFT 0x187 /* shifted left arrow key */
-#define KEY_SMESSAGE 0x188 /* shifted message key */
-#define KEY_SMOVE 0x189 /* shifted move key */
-#define KEY_SNEXT 0x18a /* shifted next key */
-#define KEY_SOPTIONS 0x18b /* shifted options key */
-#define KEY_SPREVIOUS 0x18c /* shifted prev key */
-#define KEY_SPRINT 0x18d /* shifted print key */
-#define KEY_SREDO 0x18e /* shifted redo key */
-#define KEY_SREPLACE 0x18f /* shifted replace key */
-#define KEY_SRIGHT 0x190 /* shifted right arrow */
-#define KEY_SRSUME 0x191 /* shifted resume key */
-#define KEY_SSAVE 0x192 /* shifted save key */
-#define KEY_SSUSPEND 0x193 /* shifted suspend key */
-#define KEY_SUNDO 0x194 /* shifted undo key */
-#define KEY_SUSPEND 0x195 /* suspend key */
-#define KEY_UNDO 0x196 /* undo key */
-
-/* PDCurses-specific key definitions -- PC only */
-
-#define ALT_0 0x197
-#define ALT_1 0x198
-#define ALT_2 0x199
-#define ALT_3 0x19a
-#define ALT_4 0x19b
-#define ALT_5 0x19c
-#define ALT_6 0x19d
-#define ALT_7 0x19e
-#define ALT_8 0x19f
-#define ALT_9 0x1a0
-#define ALT_A 0x1a1
-#define ALT_B 0x1a2
-#define ALT_C 0x1a3
-#define ALT_D 0x1a4
-#define ALT_E 0x1a5
-#define ALT_F 0x1a6
-#define ALT_G 0x1a7
-#define ALT_H 0x1a8
-#define ALT_I 0x1a9
-#define ALT_J 0x1aa
-#define ALT_K 0x1ab
-#define ALT_L 0x1ac
-#define ALT_M 0x1ad
-#define ALT_N 0x1ae
-#define ALT_O 0x1af
-#define ALT_P 0x1b0
-#define ALT_Q 0x1b1
-#define ALT_R 0x1b2
-#define ALT_S 0x1b3
-#define ALT_T 0x1b4
-#define ALT_U 0x1b5
-#define ALT_V 0x1b6
-#define ALT_W 0x1b7
-#define ALT_X 0x1b8
-#define ALT_Y 0x1b9
-#define ALT_Z 0x1ba
-
-#define CTL_LEFT 0x1bb /* Control-Left-Arrow */
-#define CTL_RIGHT 0x1bc
-#define CTL_PGUP 0x1bd
-#define CTL_PGDN 0x1be
-#define CTL_HOME 0x1bf
-#define CTL_END 0x1c0
-
-#define KEY_A1 0x1c1 /* upper left on Virtual keypad */
-#define KEY_A2 0x1c2 /* upper middle on Virt. keypad */
-#define KEY_A3 0x1c3 /* upper right on Vir. keypad */
-#define KEY_B1 0x1c4 /* middle left on Virt. keypad */
-#define KEY_B2 0x1c5 /* center on Virt. keypad */
-#define KEY_B3 0x1c6 /* middle right on Vir. keypad */
-#define KEY_C1 0x1c7 /* lower left on Virt. keypad */
-#define KEY_C2 0x1c8 /* lower middle on Virt. keypad */
-#define KEY_C3 0x1c9 /* lower right on Vir. keypad */
-
-#define PADSLASH 0x1ca /* slash on keypad */
-#define PADENTER 0x1cb /* enter on keypad */
-#define CTL_PADENTER 0x1cc /* ctl-enter on keypad */
-#define ALT_PADENTER 0x1cd /* alt-enter on keypad */
-#define PADSTOP 0x1ce /* stop on keypad */
-#define PADSTAR 0x1cf /* star on keypad */
-#define PADMINUS 0x1d0 /* minus on keypad */
-#define PADPLUS 0x1d1 /* plus on keypad */
-#define CTL_PADSTOP 0x1d2 /* ctl-stop on keypad */
-#define CTL_PADCENTER 0x1d3 /* ctl-enter on keypad */
-#define CTL_PADPLUS 0x1d4 /* ctl-plus on keypad */
-#define CTL_PADMINUS 0x1d5 /* ctl-minus on keypad */
-#define CTL_PADSLASH 0x1d6 /* ctl-slash on keypad */
-#define CTL_PADSTAR 0x1d7 /* ctl-star on keypad */
-#define ALT_PADPLUS 0x1d8 /* alt-plus on keypad */
-#define ALT_PADMINUS 0x1d9 /* alt-minus on keypad */
-#define ALT_PADSLASH 0x1da /* alt-slash on keypad */
-#define ALT_PADSTAR 0x1db /* alt-star on keypad */
-#define ALT_PADSTOP 0x1dc /* alt-stop on keypad */
-#define CTL_INS 0x1dd /* ctl-insert */
-#define ALT_DEL 0x1de /* alt-delete */
-#define ALT_INS 0x1df /* alt-insert */
-#define CTL_UP 0x1e0 /* ctl-up arrow */
-#define CTL_DOWN 0x1e1 /* ctl-down arrow */
-#define CTL_TAB 0x1e2 /* ctl-tab */
-#define ALT_TAB 0x1e3
-#define ALT_MINUS 0x1e4
-#define ALT_EQUAL 0x1e5
-#define ALT_HOME 0x1e6
-#define ALT_PGUP 0x1e7
-#define ALT_PGDN 0x1e8
-#define ALT_END 0x1e9
-#define ALT_UP 0x1ea /* alt-up arrow */
-#define ALT_DOWN 0x1eb /* alt-down arrow */
-#define ALT_RIGHT 0x1ec /* alt-right arrow */
-#define ALT_LEFT 0x1ed /* alt-left arrow */
-#define ALT_ENTER 0x1ee /* alt-enter */
-#define ALT_ESC 0x1ef /* alt-escape */
-#define ALT_BQUOTE 0x1f0 /* alt-back quote */
-#define ALT_LBRACKET 0x1f1 /* alt-left bracket */
-#define ALT_RBRACKET 0x1f2 /* alt-right bracket */
-#define ALT_SEMICOLON 0x1f3 /* alt-semi-colon */
-#define ALT_FQUOTE 0x1f4 /* alt-forward quote */
-#define ALT_COMMA 0x1f5 /* alt-comma */
-#define ALT_STOP 0x1f6 /* alt-stop */
-#define ALT_FSLASH 0x1f7 /* alt-forward slash */
-#define ALT_BKSP 0x1f8 /* alt-backspace */
-#define CTL_BKSP 0x1f9 /* ctl-backspace */
-#define PAD0 0x1fa /* keypad 0 */
-
-#define CTL_PAD0 0x1fb /* ctl-keypad 0 */
-#define CTL_PAD1 0x1fc
-#define CTL_PAD2 0x1fd
-#define CTL_PAD3 0x1fe
-#define CTL_PAD4 0x1ff
-#define CTL_PAD5 0x200
-#define CTL_PAD6 0x201
-#define CTL_PAD7 0x202
-#define CTL_PAD8 0x203
-#define CTL_PAD9 0x204
-
-#define ALT_PAD0 0x205 /* alt-keypad 0 */
-#define ALT_PAD1 0x206
-#define ALT_PAD2 0x207
-#define ALT_PAD3 0x208
-#define ALT_PAD4 0x209
-#define ALT_PAD5 0x20a
-#define ALT_PAD6 0x20b
-#define ALT_PAD7 0x20c
-#define ALT_PAD8 0x20d
-#define ALT_PAD9 0x20e
-
-#define CTL_DEL 0x20f /* clt-delete */
-#define ALT_BSLASH 0x210 /* alt-back slash */
-#define CTL_ENTER 0x211 /* ctl-enter */
-
-#define SHF_PADENTER 0x212 /* shift-enter on keypad */
-#define SHF_PADSLASH 0x213 /* shift-slash on keypad */
-#define SHF_PADSTAR 0x214 /* shift-star on keypad */
-#define SHF_PADPLUS 0x215 /* shift-plus on keypad */
-#define SHF_PADMINUS 0x216 /* shift-minus on keypad */
-#define SHF_UP 0x217 /* shift-up on keypad */
-#define SHF_DOWN 0x218 /* shift-down on keypad */
-#define SHF_IC 0x219 /* shift-insert on keypad */
-#define SHF_DC 0x21a /* shift-delete on keypad */
-
-#define KEY_MOUSE 0x21b /* "mouse" key */
-#define KEY_SHIFT_L 0x21c /* Left-shift */
-#define KEY_SHIFT_R 0x21d /* Right-shift */
-#define KEY_CONTROL_L 0x21e /* Left-control */
-#define KEY_CONTROL_R 0x21f /* Right-control */
-#define KEY_ALT_L 0x220 /* Left-alt */
-#define KEY_ALT_R 0x221 /* Right-alt */
-#define KEY_RESIZE 0x222 /* Window resize */
-#define KEY_SUP 0x223 /* Shifted up arrow */
-#define KEY_SDOWN 0x224 /* Shifted down arrow */
-
-#define KEY_MIN KEY_BREAK /* Minimum curses key value */
-#define KEY_MAX KEY_SDOWN /* Maximum curses key */
-
-#define KEY_F(n) (KEY_F0 + (n))
-
-/*----------------------------------------------------------------------
- *
- * PDCurses Function Declarations
- *
- */
-
-/* Standard */
-
-int addch(const chtype);
-int addchnstr(const chtype *, int);
-int addchstr(const chtype *);
-int addnstr(const char *, int);
-int addstr(const char *);
-int attroff(chtype);
-int attron(chtype);
-int attrset(chtype);
-int attr_get(attr_t *, short *, void *);
-int attr_off(attr_t, void *);
-int attr_on(attr_t, void *);
-int attr_set(attr_t, short, void *);
-int baudrate(void);
-int beep(void);
-int bkgd(chtype);
-void bkgdset(chtype);
-int border(chtype, chtype, chtype, chtype, chtype, chtype, chtype, chtype);
-int box(WINDOW *, chtype, chtype);
-bool can_change_color(void);
-int cbreak(void);
-int chgat(int, attr_t, short, const void *);
-int clearok(WINDOW *, bool);
-int clear(void);
-int clrtobot(void);
-int clrtoeol(void);
-int color_content(short, short *, short *, short *);
-int color_set(short, void *);
-int copywin(const WINDOW *, WINDOW *, int, int, int, int, int, int, int);
-int curs_set(int);
-int def_prog_mode(void);
-int def_shell_mode(void);
-int delay_output(int);
-int delch(void);
-int deleteln(void);
-void delscreen(SCREEN *);
-int delwin(WINDOW *);
-WINDOW *derwin(WINDOW *, int, int, int, int);
-int doupdate(void);
-WINDOW *dupwin(WINDOW *);
-int echochar(const chtype);
-int echo(void);
-int endwin(void);
-char erasechar(void);
-int erase(void);
-void filter(void);
-int flash(void);
-int flushinp(void);
-chtype getbkgd(WINDOW *);
-int getnstr(char *, int);
-int getstr(char *);
-WINDOW *getwin(FILE *);
-int halfdelay(int);
-bool has_colors(void);
-bool has_ic(void);
-bool has_il(void);
-int hline(chtype, int);
-void idcok(WINDOW *, bool);
-int idlok(WINDOW *, bool);
-void immedok(WINDOW *, bool);
-int inchnstr(chtype *, int);
-int inchstr(chtype *);
-chtype inch(void);
-int init_color(short, short, short, short);
-int init_pair(short, short, short);
-WINDOW *initscr(void);
-int innstr(char *, int);
-int insch(chtype);
-int insdelln(int);
-int insertln(void);
-int insnstr(const char *, int);
-int insstr(const char *);
-int instr(char *);
-int intrflush(WINDOW *, bool);
-bool isendwin(void);
-bool is_linetouched(WINDOW *, int);
-bool is_wintouched(WINDOW *);
-char *keyname(int);
-int keypad(WINDOW *, bool);
-char killchar(void);
-int leaveok(WINDOW *, bool);
-char *longname(void);
-int meta(WINDOW *, bool);
-int move(int, int);
-int mvaddch(int, int, const chtype);
-int mvaddchnstr(int, int, const chtype *, int);
-int mvaddchstr(int, int, const chtype *);
-int mvaddnstr(int, int, const char *, int);
-int mvaddstr(int, int, const char *);
-int mvchgat(int, int, int, attr_t, short, const void *);
-int mvcur(int, int, int, int);
-int mvdelch(int, int);
-int mvderwin(WINDOW *, int, int);
-int mvgetch(int, int);
-int mvgetnstr(int, int, char *, int);
-int mvgetstr(int, int, char *);
-int mvhline(int, int, chtype, int);
-chtype mvinch(int, int);
-int mvinchnstr(int, int, chtype *, int);
-int mvinchstr(int, int, chtype *);
-int mvinnstr(int, int, char *, int);
-int mvinsch(int, int, chtype);
-int mvinsnstr(int, int, const char *, int);
-int mvinsstr(int, int, const char *);
-int mvinstr(int, int, char *);
-int mvprintw(int, int, const char *, ...);
-int mvscanw(int, int, const char *, ...);
-int mvvline(int, int, chtype, int);
-int mvwaddchnstr(WINDOW *, int, int, const chtype *, int);
-int mvwaddchstr(WINDOW *, int, int, const chtype *);
-int mvwaddch(WINDOW *, int, int, const chtype);
-int mvwaddnstr(WINDOW *, int, int, const char *, int);
-int mvwaddstr(WINDOW *, int, int, const char *);
-int mvwchgat(WINDOW *, int, int, int, attr_t, short, const void *);
-int mvwdelch(WINDOW *, int, int);
-int mvwgetch(WINDOW *, int, int);
-int mvwgetnstr(WINDOW *, int, int, char *, int);
-int mvwgetstr(WINDOW *, int, int, char *);
-int mvwhline(WINDOW *, int, int, chtype, int);
-int mvwinchnstr(WINDOW *, int, int, chtype *, int);
-int mvwinchstr(WINDOW *, int, int, chtype *);
-chtype mvwinch(WINDOW *, int, int);
-int mvwinnstr(WINDOW *, int, int, char *, int);
-int mvwinsch(WINDOW *, int, int, chtype);
-int mvwinsnstr(WINDOW *, int, int, const char *, int);
-int mvwinsstr(WINDOW *, int, int, const char *);
-int mvwinstr(WINDOW *, int, int, char *);
-int mvwin(WINDOW *, int, int);
-int mvwprintw(WINDOW *, int, int, const char *, ...);
-int mvwscanw(WINDOW *, int, int, const char *, ...);
-int mvwvline(WINDOW *, int, int, chtype, int);
-int napms(int);
-WINDOW *newpad(int, int);
-SCREEN *newterm(const char *, FILE *, FILE *);
-WINDOW *newwin(int, int, int, int);
-int nl(void);
-int nocbreak(void);
-int nodelay(WINDOW *, bool);
-int noecho(void);
-int nonl(void);
-void noqiflush(void);
-int noraw(void);
-int notimeout(WINDOW *, bool);
-int overlay(const WINDOW *, WINDOW *);
-int overwrite(const WINDOW *, WINDOW *);
-int pair_content(short, short *, short *);
-int pechochar(WINDOW *, chtype);
-int pnoutrefresh(WINDOW *, int, int, int, int, int, int);
-int prefresh(WINDOW *, int, int, int, int, int, int);
-int printw(const char *, ...);
-int putwin(WINDOW *, FILE *);
-void qiflush(void);
-int raw(void);
-int redrawwin(WINDOW *);
-int refresh(void);
-int reset_prog_mode(void);
-int reset_shell_mode(void);
-int resetty(void);
-int ripoffline(int, int (*)(WINDOW *, int));
-int savetty(void);
-int scanw(const char *, ...);
-int scr_dump(const char *);
-int scr_init(const char *);
-int scr_restore(const char *);
-int scr_set(const char *);
-int scrl(int);
-int scroll(WINDOW *);
-int scrollok(WINDOW *, bool);
-SCREEN *set_term(SCREEN *);
-int setscrreg(int, int);
-int slk_attroff(const chtype);
-int slk_attr_off(const attr_t, void *);
-int slk_attron(const chtype);
-int slk_attr_on(const attr_t, void *);
-int slk_attrset(const chtype);
-int slk_attr_set(const attr_t, short, void *);
-int slk_clear(void);
-int slk_color(short);
-int slk_init(int);
-char *slk_label(int);
-int slk_noutrefresh(void);
-int slk_refresh(void);
-int slk_restore(void);
-int slk_set(int, const char *, int);
-int slk_touch(void);
-int standend(void);
-int standout(void);
-int start_color(void);
-WINDOW *subpad(WINDOW *, int, int, int, int);
-WINDOW *subwin(WINDOW *, int, int, int, int);
-int syncok(WINDOW *, bool);
-chtype termattrs(void);
-attr_t term_attrs(void);
-char *termname(void);
-void timeout(int);
-int touchline(WINDOW *, int, int);
-int touchwin(WINDOW *);
-int typeahead(int);
-int untouchwin(WINDOW *);
-void use_env(bool);
-int vidattr(chtype);
-int vid_attr(attr_t, short, void *);
-int vidputs(chtype, int (*)(int));
-int vid_puts(attr_t, short, void *, int (*)(int));
-int vline(chtype, int);
-int vw_printw(WINDOW *, const char *, va_list);
-int vwprintw(WINDOW *, const char *, va_list);
-int vw_scanw(WINDOW *, const char *, va_list);
-int vwscanw(WINDOW *, const char *, va_list);
-int waddchnstr(WINDOW *, const chtype *, int);
-int waddchstr(WINDOW *, const chtype *);
-int waddch(WINDOW *, const chtype);
-int waddnstr(WINDOW *, const char *, int);
-int waddstr(WINDOW *, const char *);
-int wattroff(WINDOW *, chtype);
-int wattron(WINDOW *, chtype);
-int wattrset(WINDOW *, chtype);
-int wattr_get(WINDOW *, attr_t *, short *, void *);
-int wattr_off(WINDOW *, attr_t, void *);
-int wattr_on(WINDOW *, attr_t, void *);
-int wattr_set(WINDOW *, attr_t, short, void *);
-void wbkgdset(WINDOW *, chtype);
-int wbkgd(WINDOW *, chtype);
-int wborder(WINDOW *, chtype, chtype, chtype, chtype,
- chtype, chtype, chtype, chtype);
-int wchgat(WINDOW *, int, attr_t, short, const void *);
-int wclear(WINDOW *);
-int wclrtobot(WINDOW *);
-int wclrtoeol(WINDOW *);
-int wcolor_set(WINDOW *, short, void *);
-void wcursyncup(WINDOW *);
-int wdelch(WINDOW *);
-int wdeleteln(WINDOW *);
-int wechochar(WINDOW *, const chtype);
-int werase(WINDOW *);
-int wgetch(WINDOW *);
-int wgetnstr(WINDOW *, char *, int);
-int wgetstr(WINDOW *, char *);
-int whline(WINDOW *, chtype, int);
-int winchnstr(WINDOW *, chtype *, int);
-int winchstr(WINDOW *, chtype *);
-chtype winch(WINDOW *);
-int winnstr(WINDOW *, char *, int);
-int winsch(WINDOW *, chtype);
-int winsdelln(WINDOW *, int);
-int winsertln(WINDOW *);
-int winsnstr(WINDOW *, const char *, int);
-int winsstr(WINDOW *, const char *);
-int winstr(WINDOW *, char *);
-int wmove(WINDOW *, int, int);
-int wnoutrefresh(WINDOW *);
-int wprintw(WINDOW *, const char *, ...);
-int wredrawln(WINDOW *, int, int);
-int wrefresh(WINDOW *);
-int wscanw(WINDOW *, const char *, ...);
-int wscrl(WINDOW *, int);
-int wsetscrreg(WINDOW *, int, int);
-int wstandend(WINDOW *);
-int wstandout(WINDOW *);
-void wsyncdown(WINDOW *);
-void wsyncup(WINDOW *);
-void wtimeout(WINDOW *, int);
-int wtouchln(WINDOW *, int, int, int);
-int wvline(WINDOW *, chtype, int);
-
-/* Wide-character functions */
-
-#ifdef PDC_WIDE
-int addnwstr(const wchar_t *, int);
-int addwstr(const wchar_t *);
-int add_wch(const cchar_t *);
-int add_wchnstr(const cchar_t *, int);
-int add_wchstr(const cchar_t *);
-int border_set(const cchar_t *, const cchar_t *, const cchar_t *,
- const cchar_t *, const cchar_t *, const cchar_t *,
- const cchar_t *, const cchar_t *);
-int box_set(WINDOW *, const cchar_t *, const cchar_t *);
-int echo_wchar(const cchar_t *);
-int erasewchar(wchar_t *);
-int getbkgrnd(cchar_t *);
-int getcchar(const cchar_t *, wchar_t *, attr_t *, short *, void *);
-int getn_wstr(wint_t *, int);
-int get_wch(wint_t *);
-int get_wstr(wint_t *);
-int hline_set(const cchar_t *, int);
-int innwstr(wchar_t *, int);
-int ins_nwstr(const wchar_t *, int);
-int ins_wch(const cchar_t *);
-int ins_wstr(const wchar_t *);
-int inwstr(wchar_t *);
-int in_wch(cchar_t *);
-int in_wchnstr(cchar_t *, int);
-int in_wchstr(cchar_t *);
-char *key_name(wchar_t);
-int killwchar(wchar_t *);
-int mvaddnwstr(int, int, const wchar_t *, int);
-int mvaddwstr(int, int, const wchar_t *);
-int mvadd_wch(int, int, const cchar_t *);
-int mvadd_wchnstr(int, int, const cchar_t *, int);
-int mvadd_wchstr(int, int, const cchar_t *);
-int mvgetn_wstr(int, int, wint_t *, int);
-int mvget_wch(int, int, wint_t *);
-int mvget_wstr(int, int, wint_t *);
-int mvhline_set(int, int, const cchar_t *, int);
-int mvinnwstr(int, int, wchar_t *, int);
-int mvins_nwstr(int, int, const wchar_t *, int);
-int mvins_wch(int, int, const cchar_t *);
-int mvins_wstr(int, int, const wchar_t *);
-int mvinwstr(int, int, wchar_t *);
-int mvin_wch(int, int, cchar_t *);
-int mvin_wchnstr(int, int, cchar_t *, int);
-int mvin_wchstr(int, int, cchar_t *);
-int mvvline_set(int, int, const cchar_t *, int);
-int mvwaddnwstr(WINDOW *, int, int, const wchar_t *, int);
-int mvwaddwstr(WINDOW *, int, int, const wchar_t *);
-int mvwadd_wch(WINDOW *, int, int, const cchar_t *);
-int mvwadd_wchnstr(WINDOW *, int, int, const cchar_t *, int);
-int mvwadd_wchstr(WINDOW *, int, int, const cchar_t *);
-int mvwgetn_wstr(WINDOW *, int, int, wint_t *, int);
-int mvwget_wch(WINDOW *, int, int, wint_t *);
-int mvwget_wstr(WINDOW *, int, int, wint_t *);
-int mvwhline_set(WINDOW *, int, int, const cchar_t *, int);
-int mvwinnwstr(WINDOW *, int, int, wchar_t *, int);
-int mvwins_nwstr(WINDOW *, int, int, const wchar_t *, int);
-int mvwins_wch(WINDOW *, int, int, const cchar_t *);
-int mvwins_wstr(WINDOW *, int, int, const wchar_t *);
-int mvwin_wch(WINDOW *, int, int, cchar_t *);
-int mvwin_wchnstr(WINDOW *, int, int, cchar_t *, int);
-int mvwin_wchstr(WINDOW *, int, int, cchar_t *);
-int mvwinwstr(WINDOW *, int, int, wchar_t *);
-int mvwvline_set(WINDOW *, int, int, const cchar_t *, int);
-int pecho_wchar(WINDOW *, const cchar_t*);
-int setcchar(cchar_t*, const wchar_t*, const attr_t, short, const void*);
-int slk_wset(int, const wchar_t *, int);
-int unget_wch(const wchar_t);
-int vline_set(const cchar_t *, int);
-int waddnwstr(WINDOW *, const wchar_t *, int);
-int waddwstr(WINDOW *, const wchar_t *);
-int wadd_wch(WINDOW *, const cchar_t *);
-int wadd_wchnstr(WINDOW *, const cchar_t *, int);
-int wadd_wchstr(WINDOW *, const cchar_t *);
-int wbkgrnd(WINDOW *, const cchar_t *);
-void wbkgrndset(WINDOW *, const cchar_t *);
-int wborder_set(WINDOW *, const cchar_t *, const cchar_t *,
- const cchar_t *, const cchar_t *, const cchar_t *,
- const cchar_t *, const cchar_t *, const cchar_t *);
-int wecho_wchar(WINDOW *, const cchar_t *);
-int wgetbkgrnd(WINDOW *, cchar_t *);
-int wgetn_wstr(WINDOW *, wint_t *, int);
-int wget_wch(WINDOW *, wint_t *);
-int wget_wstr(WINDOW *, wint_t *);
-int whline_set(WINDOW *, const cchar_t *, int);
-int winnwstr(WINDOW *, wchar_t *, int);
-int wins_nwstr(WINDOW *, const wchar_t *, int);
-int wins_wch(WINDOW *, const cchar_t *);
-int wins_wstr(WINDOW *, const wchar_t *);
-int winwstr(WINDOW *, wchar_t *);
-int win_wch(WINDOW *, cchar_t *);
-int win_wchnstr(WINDOW *, cchar_t *, int);
-int win_wchstr(WINDOW *, cchar_t *);
-wchar_t *wunctrl(cchar_t *);
-int wvline_set(WINDOW *, const cchar_t *, int);
-#endif
-
-/* Quasi-standard */
-
-chtype getattrs(WINDOW *);
-int getbegx(WINDOW *);
-int getbegy(WINDOW *);
-int getmaxx(WINDOW *);
-int getmaxy(WINDOW *);
-int getparx(WINDOW *);
-int getpary(WINDOW *);
-int getcurx(WINDOW *);
-int getcury(WINDOW *);
-void traceoff(void);
-void traceon(void);
-char *unctrl(chtype);
-
-int crmode(void);
-int nocrmode(void);
-int draino(int);
-int resetterm(void);
-int fixterm(void);
-int saveterm(void);
-int setsyx(int, int);
-
-int mouse_set(unsigned long);
-int mouse_on(unsigned long);
-int mouse_off(unsigned long);
-int request_mouse_pos(void);
-int map_button(unsigned long);
-void wmouse_position(WINDOW *, int *, int *);
-unsigned long getmouse(void);
-unsigned long getbmap(void);
-
-/* ncurses */
-
-int assume_default_colors(int, int);
-const char *curses_version(void);
-bool has_key(int);
-int use_default_colors(void);
-int wresize(WINDOW *, int, int);
-
-int mouseinterval(int);
-mmask_t mousemask(mmask_t, mmask_t *);
-bool mouse_trafo(int *, int *, bool);
-int nc_getmouse(MEVENT *);
-int ungetmouse(MEVENT *);
-bool wenclose(const WINDOW *, int, int);
-bool wmouse_trafo(const WINDOW *, int *, int *, bool);
-
-/* PDCurses */
-
-int addrawch(chtype);
-int insrawch(chtype);
-bool is_termresized(void);
-int mvaddrawch(int, int, chtype);
-int mvdeleteln(int, int);
-int mvinsertln(int, int);
-int mvinsrawch(int, int, chtype);
-int mvwaddrawch(WINDOW *, int, int, chtype);
-int mvwdeleteln(WINDOW *, int, int);
-int mvwinsertln(WINDOW *, int, int);
-int mvwinsrawch(WINDOW *, int, int, chtype);
-int raw_output(bool);
-int resize_term(int, int);
-WINDOW *resize_window(WINDOW *, int, int);
-int waddrawch(WINDOW *, chtype);
-int winsrawch(WINDOW *, chtype);
-char wordchar(void);
-
-#ifdef PDC_WIDE
-wchar_t *slk_wlabel(int);
-#endif
-
-void PDC_debug(const char *, ...);
-int PDC_ungetch(int);
-int PDC_set_blink(bool);
-int PDC_set_line_color(short);
-void PDC_set_title(const char *);
-
-int PDC_clearclipboard(void);
-int PDC_freeclipboard(char *);
-int PDC_getclipboard(char **, long *);
-int PDC_setclipboard(const char *, long);
-
-unsigned long PDC_get_input_fd(void);
-unsigned long PDC_get_key_modifiers(void);
-int PDC_return_key_modifiers(bool);
-int PDC_save_key_modifiers(bool);
-
-#ifdef XCURSES
-WINDOW *Xinitscr(int, char **);
-void XCursesExit(void);
-int sb_init(void);
-int sb_set_horz(int, int, int);
-int sb_set_vert(int, int, int);
-int sb_get_horz(int *, int *, int *);
-int sb_get_vert(int *, int *, int *);
-int sb_refresh(void);
-#endif
-
-/*** Functions defined as macros ***/
-
-/* getch() and ungetch() conflict with some DOS libraries */
-
-#define getch() wgetch(stdscr)
-#define ungetch(ch) PDC_ungetch(ch)
-
-#define COLOR_PAIR(n) (((chtype)(n) << PDC_COLOR_SHIFT) & A_COLOR)
-#define PAIR_NUMBER(n) (((n) & A_COLOR) >> PDC_COLOR_SHIFT)
-
-/* These will _only_ work as macros */
-
-#define getbegyx(w, y, x) (y = getbegy(w), x = getbegx(w))
-#define getmaxyx(w, y, x) (y = getmaxy(w), x = getmaxx(w))
-#define getparyx(w, y, x) (y = getpary(w), x = getparx(w))
-#define getyx(w, y, x) (y = getcury(w), x = getcurx(w))
-
-#define getsyx(y, x) { if (curscr->_leaveit) (y)=(x)=-1; \
- else getyx(curscr,(y),(x)); }
-
-#ifdef NCURSES_MOUSE_VERSION
-# define getmouse(x) nc_getmouse(x)
-#endif
-
-/* return codes from PDC_getclipboard() and PDC_setclipboard() calls */
-
-#define PDC_CLIP_SUCCESS 0
-#define PDC_CLIP_ACCESS_ERROR 1
-#define PDC_CLIP_EMPTY 2
-#define PDC_CLIP_MEMORY_ERROR 3
-
-/* PDCurses key modifier masks */
-
-#define PDC_KEY_MODIFIER_SHIFT 1
-#define PDC_KEY_MODIFIER_CONTROL 2
-#define PDC_KEY_MODIFIER_ALT 4
-#define PDC_KEY_MODIFIER_NUMLOCK 8
-
-#if defined(__cplusplus) || defined(__cplusplus__) || defined(__CPLUSPLUS)
-# undef bool
-}
-#endif
-
-#endif /* __PDCURSES__ */
+++ /dev/null
-/* zconf.h -- configuration of the zlib compression library
- * Copyright (C) 1995-2005 Jean-loup Gailly.
- * For conditions of distribution and use, see copyright notice in zlib.h
- */
-
-/* @(#) $Id$ */
-
-#ifndef ZCONF_H
-#define ZCONF_H
-
-/*
- * If you *really* need a unique prefix for all types and library functions,
- * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it.
- */
-#ifdef Z_PREFIX
-# define deflateInit_ z_deflateInit_
-# define deflate z_deflate
-# define deflateEnd z_deflateEnd
-# define inflateInit_ z_inflateInit_
-# define inflate z_inflate
-# define inflateEnd z_inflateEnd
-# define deflateInit2_ z_deflateInit2_
-# define deflateSetDictionary z_deflateSetDictionary
-# define deflateCopy z_deflateCopy
-# define deflateReset z_deflateReset
-# define deflateParams z_deflateParams
-# define deflateBound z_deflateBound
-# define deflatePrime z_deflatePrime
-# define inflateInit2_ z_inflateInit2_
-# define inflateSetDictionary z_inflateSetDictionary
-# define inflateSync z_inflateSync
-# define inflateSyncPoint z_inflateSyncPoint
-# define inflateCopy z_inflateCopy
-# define inflateReset z_inflateReset
-# define inflateBack z_inflateBack
-# define inflateBackEnd z_inflateBackEnd
-# define compress z_compress
-# define compress2 z_compress2
-# define compressBound z_compressBound
-# define uncompress z_uncompress
-# define adler32 z_adler32
-# define crc32 z_crc32
-# define get_crc_table z_get_crc_table
-# define zError z_zError
-
-# define alloc_func z_alloc_func
-# define free_func z_free_func
-# define in_func z_in_func
-# define out_func z_out_func
-# define Byte z_Byte
-# define uInt z_uInt
-# define uLong z_uLong
-# define Bytef z_Bytef
-# define charf z_charf
-# define intf z_intf
-# define uIntf z_uIntf
-# define uLongf z_uLongf
-# define voidpf z_voidpf
-# define voidp z_voidp
-#endif
-
-#if defined(__MSDOS__) && !defined(MSDOS)
-# define MSDOS
-#endif
-#if (defined(OS_2) || defined(__OS2__)) && !defined(OS2)
-# define OS2
-#endif
-#if defined(_WINDOWS) && !defined(WINDOWS)
-# define WINDOWS
-#endif
-#if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__)
-# ifndef WIN32
-# define WIN32
-# endif
-#endif
-#if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32)
-# if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__)
-# ifndef SYS16BIT
-# define SYS16BIT
-# endif
-# endif
-#endif
-
-/*
- * Compile with -DMAXSEG_64K if the alloc function cannot allocate more
- * than 64k bytes at a time (needed on systems with 16-bit int).
- */
-#ifdef SYS16BIT
-# define MAXSEG_64K
-#endif
-#ifdef MSDOS
-# define UNALIGNED_OK
-#endif
-
-#ifdef __STDC_VERSION__
-# ifndef STDC
-# define STDC
-# endif
-# if __STDC_VERSION__ >= 199901L
-# ifndef STDC99
-# define STDC99
-# endif
-# endif
-#endif
-#if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus))
-# define STDC
-#endif
-#if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__))
-# define STDC
-#endif
-#if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32))
-# define STDC
-#endif
-#if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__))
-# define STDC
-#endif
-
-#if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */
-# define STDC
-#endif
-
-#ifndef STDC
-# ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */
-# define const /* note: need a more gentle solution here */
-# endif
-#endif
-
-/* Some Mac compilers merge all .h files incorrectly: */
-#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__)
-# define NO_DUMMY_DECL
-#endif
-
-/* Maximum value for memLevel in deflateInit2 */
-#ifndef MAX_MEM_LEVEL
-# ifdef MAXSEG_64K
-# define MAX_MEM_LEVEL 8
-# else
-# define MAX_MEM_LEVEL 9
-# endif
-#endif
-
-/* Maximum value for windowBits in deflateInit2 and inflateInit2.
- * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files
- * created by gzip. (Files created by minigzip can still be extracted by
- * gzip.)
- */
-#ifndef MAX_WBITS
-# define MAX_WBITS 15 /* 32K LZ77 window */
-#endif
-
-/* The memory requirements for deflate are (in bytes):
- (1 << (windowBits+2)) + (1 << (memLevel+9))
- that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values)
- plus a few kilobytes for small objects. For example, if you want to reduce
- the default memory requirements from 256K to 128K, compile with
- make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7"
- Of course this will generally degrade compression (there's no free lunch).
-
- The memory requirements for inflate are (in bytes) 1 << windowBits
- that is, 32K for windowBits=15 (default value) plus a few kilobytes
- for small objects.
-*/
-
- /* Type declarations */
-
-#ifndef OF /* function prototypes */
-# ifdef STDC
-# define OF(args) args
-# else
-# define OF(args) ()
-# endif
-#endif
-
-/* The following definitions for FAR are needed only for MSDOS mixed
- * model programming (small or medium model with some far allocations).
- * This was tested only with MSC; for other MSDOS compilers you may have
- * to define NO_MEMCPY in zutil.h. If you don't need the mixed model,
- * just define FAR to be empty.
- */
-#ifdef SYS16BIT
-# if defined(M_I86SM) || defined(M_I86MM)
- /* MSC small or medium model */
-# define SMALL_MEDIUM
-# ifdef _MSC_VER
-# define FAR _far
-# else
-# define FAR far
-# endif
-# endif
-# if (defined(__SMALL__) || defined(__MEDIUM__))
- /* Turbo C small or medium model */
-# define SMALL_MEDIUM
-# ifdef __BORLANDC__
-# define FAR _far
-# else
-# define FAR far
-# endif
-# endif
-#endif
-
-#if defined(WINDOWS) || defined(WIN32)
- /* If building or using zlib as a DLL, define ZLIB_DLL.
- * This is not mandatory, but it offers a little performance increase.
- */
-# ifdef ZLIB_DLL
-# if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500))
-# ifdef ZLIB_INTERNAL
-# define ZEXTERN extern __declspec(dllexport)
-# else
-# define ZEXTERN extern __declspec(dllimport)
-# endif
-# endif
-# endif /* ZLIB_DLL */
- /* If building or using zlib with the WINAPI/WINAPIV calling convention,
- * define ZLIB_WINAPI.
- * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI.
- */
-# ifdef ZLIB_WINAPI
-# ifdef FAR
-# undef FAR
-# endif
-# include <windows.h>
- /* No need for _export, use ZLIB.DEF instead. */
- /* For complete Windows compatibility, use WINAPI, not __stdcall. */
-# define ZEXPORT WINAPI
-# ifdef WIN32
-# define ZEXPORTVA WINAPIV
-# else
-# define ZEXPORTVA FAR CDECL
-# endif
-# endif
-#endif
-
-#if defined (__BEOS__)
-# ifdef ZLIB_DLL
-# ifdef ZLIB_INTERNAL
-# define ZEXPORT __declspec(dllexport)
-# define ZEXPORTVA __declspec(dllexport)
-# else
-# define ZEXPORT __declspec(dllimport)
-# define ZEXPORTVA __declspec(dllimport)
-# endif
-# endif
-#endif
-
-#ifndef ZEXTERN
-# define ZEXTERN extern
-#endif
-#ifndef ZEXPORT
-# define ZEXPORT
-#endif
-#ifndef ZEXPORTVA
-# define ZEXPORTVA
-#endif
-
-#ifndef FAR
-# define FAR
-#endif
-
-#if !defined(__MACTYPES__)
-typedef unsigned char Byte; /* 8 bits */
-#endif
-typedef unsigned int uInt; /* 16 bits or more */
-typedef unsigned long uLong; /* 32 bits or more */
-
-#ifdef SMALL_MEDIUM
- /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */
-# define Bytef Byte FAR
-#else
- typedef Byte FAR Bytef;
-#endif
-typedef char FAR charf;
-typedef int FAR intf;
-typedef uInt FAR uIntf;
-typedef uLong FAR uLongf;
-
-#ifdef STDC
- typedef void const *voidpc;
- typedef void FAR *voidpf;
- typedef void *voidp;
-#else
- typedef Byte const *voidpc;
- typedef Byte FAR *voidpf;
- typedef Byte *voidp;
-#endif
-
-#if 0 /* HAVE_UNISTD_H -- this line is updated by ./configure */
-# include <sys/types.h> /* for off_t */
-# include <unistd.h> /* for SEEK_* and off_t */
-# ifdef VMS
-# include <unixio.h> /* for off_t */
-# endif
-# define z_off_t off_t
-#endif
-#ifndef SEEK_SET
-# define SEEK_SET 0 /* Seek from beginning of file. */
-# define SEEK_CUR 1 /* Seek from current position. */
-# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */
-#endif
-#ifndef z_off_t
-# define z_off_t long
-#endif
-
-#if defined(__OS400__)
-# define NO_vsnprintf
-#endif
-
-#if defined(__MVS__)
-# define NO_vsnprintf
-# ifdef FAR
-# undef FAR
-# endif
-#endif
-
-/* MVS linker does not support external names larger than 8 bytes */
-#if defined(__MVS__)
-# pragma map(deflateInit_,"DEIN")
-# pragma map(deflateInit2_,"DEIN2")
-# pragma map(deflateEnd,"DEEND")
-# pragma map(deflateBound,"DEBND")
-# pragma map(inflateInit_,"ININ")
-# pragma map(inflateInit2_,"ININ2")
-# pragma map(inflateEnd,"INEND")
-# pragma map(inflateSync,"INSY")
-# pragma map(inflateSetDictionary,"INSEDI")
-# pragma map(compressBound,"CMBND")
-# pragma map(inflate_table,"INTABL")
-# pragma map(inflate_fast,"INFA")
-# pragma map(inflate_copyright,"INCOPY")
-#endif
-
-#endif /* ZCONF_H */
+++ /dev/null
-/* zlib.h -- interface of the 'zlib' general purpose compression library
- version 1.2.3, July 18th, 2005
-
- Copyright (C) 1995-2005 Jean-loup Gailly and Mark Adler
-
- This software is provided 'as-is', without any express or implied
- warranty. In no event will the authors be held liable for any damages
- arising from the use of this software.
-
- Permission is granted to anyone to use this software for any purpose,
- including commercial applications, and to alter it and redistribute it
- freely, subject to the following restrictions:
-
- 1. The origin of this software must not be misrepresented; you must not
- claim that you wrote the original software. If you use this software
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
- 2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original software.
- 3. This notice may not be removed or altered from any source distribution.
-
- Jean-loup Gailly Mark Adler
- jloup@gzip.org madler@alumni.caltech.edu
-
-
- The data format used by the zlib library is described by RFCs (Request for
- Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt
- (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
-*/
-
-#ifndef ZLIB_H
-#define ZLIB_H
-
-#include "zconf.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define ZLIB_VERSION "1.2.3"
-#define ZLIB_VERNUM 0x1230
-
-/*
- The 'zlib' compression library provides in-memory compression and
- decompression functions, including integrity checks of the uncompressed
- data. This version of the library supports only one compression method
- (deflation) but other algorithms will be added later and will have the same
- stream interface.
-
- Compression can be done in a single step if the buffers are large
- enough (for example if an input file is mmap'ed), or can be done by
- repeated calls of the compression function. In the latter case, the
- application must provide more input and/or consume the output
- (providing more output space) before each call.
-
- The compressed data format used by default by the in-memory functions is
- the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped
- around a deflate stream, which is itself documented in RFC 1951.
-
- The library also supports reading and writing files in gzip (.gz) format
- with an interface similar to that of stdio using the functions that start
- with "gz". The gzip format is different from the zlib format. gzip is a
- gzip wrapper, documented in RFC 1952, wrapped around a deflate stream.
-
- This library can optionally read and write gzip streams in memory as well.
-
- The zlib format was designed to be compact and fast for use in memory
- and on communications channels. The gzip format was designed for single-
- file compression on file systems, has a larger header than zlib to maintain
- directory information, and uses a different, slower check method than zlib.
-
- The library does not install any signal handler. The decoder checks
- the consistency of the compressed data, so the library should never
- crash even in case of corrupted input.
-*/
-
-typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size));
-typedef void (*free_func) OF((voidpf opaque, voidpf address));
-
-struct internal_state;
-
-typedef struct z_stream_s {
- Bytef *next_in; /* next input byte */
- uInt avail_in; /* number of bytes available at next_in */
- uLong total_in; /* total nb of input bytes read so far */
-
- Bytef *next_out; /* next output byte should be put there */
- uInt avail_out; /* remaining free space at next_out */
- uLong total_out; /* total nb of bytes output so far */
-
- char *msg; /* last error message, NULL if no error */
- struct internal_state FAR *state; /* not visible by applications */
-
- alloc_func zalloc; /* used to allocate the internal state */
- free_func zfree; /* used to free the internal state */
- voidpf opaque; /* private data object passed to zalloc and zfree */
-
- int data_type; /* best guess about the data type: binary or text */
- uLong adler; /* adler32 value of the uncompressed data */
- uLong reserved; /* reserved for future use */
-} z_stream;
-
-typedef z_stream FAR *z_streamp;
-
-/*
- gzip header information passed to and from zlib routines. See RFC 1952
- for more details on the meanings of these fields.
-*/
-typedef struct gz_header_s {
- int text; /* true if compressed data believed to be text */
- uLong time; /* modification time */
- int xflags; /* extra flags (not used when writing a gzip file) */
- int os; /* operating system */
- Bytef *extra; /* pointer to extra field or Z_NULL if none */
- uInt extra_len; /* extra field length (valid if extra != Z_NULL) */
- uInt extra_max; /* space at extra (only when reading header) */
- Bytef *name; /* pointer to zero-terminated file name or Z_NULL */
- uInt name_max; /* space at name (only when reading header) */
- Bytef *comment; /* pointer to zero-terminated comment or Z_NULL */
- uInt comm_max; /* space at comment (only when reading header) */
- int hcrc; /* true if there was or will be a header crc */
- int done; /* true when done reading gzip header (not used
- when writing a gzip file) */
-} gz_header;
-
-typedef gz_header FAR *gz_headerp;
-
-/*
- The application must update next_in and avail_in when avail_in has
- dropped to zero. It must update next_out and avail_out when avail_out
- has dropped to zero. The application must initialize zalloc, zfree and
- opaque before calling the init function. All other fields are set by the
- compression library and must not be updated by the application.
-
- The opaque value provided by the application will be passed as the first
- parameter for calls of zalloc and zfree. This can be useful for custom
- memory management. The compression library attaches no meaning to the
- opaque value.
-
- zalloc must return Z_NULL if there is not enough memory for the object.
- If zlib is used in a multi-threaded application, zalloc and zfree must be
- thread safe.
-
- On 16-bit systems, the functions zalloc and zfree must be able to allocate
- exactly 65536 bytes, but will not be required to allocate more than this
- if the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS,
- pointers returned by zalloc for objects of exactly 65536 bytes *must*
- have their offset normalized to zero. The default allocation function
- provided by this library ensures this (see zutil.c). To reduce memory
- requirements and avoid any allocation of 64K objects, at the expense of
- compression ratio, compile the library with -DMAX_WBITS=14 (see zconf.h).
-
- The fields total_in and total_out can be used for statistics or
- progress reports. After compression, total_in holds the total size of
- the uncompressed data and may be saved for use in the decompressor
- (particularly if the decompressor wants to decompress everything in
- a single step).
-*/
-
- /* constants */
-
-#define Z_NO_FLUSH 0
-#define Z_PARTIAL_FLUSH 1 /* will be removed, use Z_SYNC_FLUSH instead */
-#define Z_SYNC_FLUSH 2
-#define Z_FULL_FLUSH 3
-#define Z_FINISH 4
-#define Z_BLOCK 5
-/* Allowed flush values; see deflate() and inflate() below for details */
-
-#define Z_OK 0
-#define Z_STREAM_END 1
-#define Z_NEED_DICT 2
-#define Z_ERRNO (-1)
-#define Z_STREAM_ERROR (-2)
-#define Z_DATA_ERROR (-3)
-#define Z_MEM_ERROR (-4)
-#define Z_BUF_ERROR (-5)
-#define Z_VERSION_ERROR (-6)
-/* Return codes for the compression/decompression functions. Negative
- * values are errors, positive values are used for special but normal events.
- */
-
-#define Z_NO_COMPRESSION 0
-#define Z_BEST_SPEED 1
-#define Z_BEST_COMPRESSION 9
-#define Z_DEFAULT_COMPRESSION (-1)
-/* compression levels */
-
-#define Z_FILTERED 1
-#define Z_HUFFMAN_ONLY 2
-#define Z_RLE 3
-#define Z_FIXED 4
-#define Z_DEFAULT_STRATEGY 0
-/* compression strategy; see deflateInit2() below for details */
-
-#define Z_BINARY 0
-#define Z_TEXT 1
-#define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */
-#define Z_UNKNOWN 2
-/* Possible values of the data_type field (though see inflate()) */
-
-#define Z_DEFLATED 8
-/* The deflate compression method (the only one supported in this version) */
-
-#define Z_NULL 0 /* for initializing zalloc, zfree, opaque */
-
-#define zlib_version zlibVersion()
-/* for compatibility with versions < 1.0.2 */
-
- /* basic functions */
-
-ZEXTERN const char * ZEXPORT zlibVersion OF((void));
-/* The application can compare zlibVersion and ZLIB_VERSION for consistency.
- If the first character differs, the library code actually used is
- not compatible with the zlib.h header file used by the application.
- This check is automatically made by deflateInit and inflateInit.
- */
-
-/*
-ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level));
-
- Initializes the internal stream state for compression. The fields
- zalloc, zfree and opaque must be initialized before by the caller.
- If zalloc and zfree are set to Z_NULL, deflateInit updates them to
- use default allocation functions.
-
- The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9:
- 1 gives best speed, 9 gives best compression, 0 gives no compression at
- all (the input data is simply copied a block at a time).
- Z_DEFAULT_COMPRESSION requests a default compromise between speed and
- compression (currently equivalent to level 6).
-
- deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not
- enough memory, Z_STREAM_ERROR if level is not a valid compression level,
- Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible
- with the version assumed by the caller (ZLIB_VERSION).
- msg is set to null if there is no error message. deflateInit does not
- perform any compression: this will be done by deflate().
-*/
-
-
-ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
-/*
- deflate compresses as much data as possible, and stops when the input
- buffer becomes empty or the output buffer becomes full. It may introduce some
- output latency (reading input without producing any output) except when
- forced to flush.
-
- The detailed semantics are as follows. deflate performs one or both of the
- following actions:
-
- - Compress more input starting at next_in and update next_in and avail_in
- accordingly. If not all input can be processed (because there is not
- enough room in the output buffer), next_in and avail_in are updated and
- processing will resume at this point for the next call of deflate().
-
- - Provide more output starting at next_out and update next_out and avail_out
- accordingly. This action is forced if the parameter flush is non zero.
- Forcing flush frequently degrades the compression ratio, so this parameter
- should be set only when necessary (in interactive applications).
- Some output may be provided even if flush is not set.
-
- Before the call of deflate(), the application should ensure that at least
- one of the actions is possible, by providing more input and/or consuming
- more output, and updating avail_in or avail_out accordingly; avail_out
- should never be zero before the call. The application can consume the
- compressed output when it wants, for example when the output buffer is full
- (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK
- and with zero avail_out, it must be called again after making room in the
- output buffer because there might be more output pending.
-
- Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to
- decide how much data to accumualte before producing output, in order to
- maximize compression.
-
- If the parameter flush is set to Z_SYNC_FLUSH, all pending output is
- flushed to the output buffer and the output is aligned on a byte boundary, so
- that the decompressor can get all input data available so far. (In particular
- avail_in is zero after the call if enough output space has been provided
- before the call.) Flushing may degrade compression for some compression
- algorithms and so it should be used only when necessary.
-
- If flush is set to Z_FULL_FLUSH, all output is flushed as with
- Z_SYNC_FLUSH, and the compression state is reset so that decompression can
- restart from this point if previous compressed data has been damaged or if
- random access is desired. Using Z_FULL_FLUSH too often can seriously degrade
- compression.
-
- If deflate returns with avail_out == 0, this function must be called again
- with the same value of the flush parameter and more output space (updated
- avail_out), until the flush is complete (deflate returns with non-zero
- avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that
- avail_out is greater than six to avoid repeated flush markers due to
- avail_out == 0 on return.
-
- If the parameter flush is set to Z_FINISH, pending input is processed,
- pending output is flushed and deflate returns with Z_STREAM_END if there
- was enough output space; if deflate returns with Z_OK, this function must be
- called again with Z_FINISH and more output space (updated avail_out) but no
- more input data, until it returns with Z_STREAM_END or an error. After
- deflate has returned Z_STREAM_END, the only possible operations on the
- stream are deflateReset or deflateEnd.
-
- Z_FINISH can be used immediately after deflateInit if all the compression
- is to be done in a single step. In this case, avail_out must be at least
- the value returned by deflateBound (see below). If deflate does not return
- Z_STREAM_END, then it must be called again as described above.
-
- deflate() sets strm->adler to the adler32 checksum of all input read
- so far (that is, total_in bytes).
-
- deflate() may update strm->data_type if it can make a good guess about
- the input data type (Z_BINARY or Z_TEXT). In doubt, the data is considered
- binary. This field is only for information purposes and does not affect
- the compression algorithm in any manner.
-
- deflate() returns Z_OK if some progress has been made (more input
- processed or more output produced), Z_STREAM_END if all input has been
- consumed and all output has been produced (only when flush is set to
- Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example
- if next_in or next_out was NULL), Z_BUF_ERROR if no progress is possible
- (for example avail_in or avail_out was zero). Note that Z_BUF_ERROR is not
- fatal, and deflate() can be called again with more input and more output
- space to continue compressing.
-*/
-
-
-ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm));
-/*
- All dynamically allocated data structures for this stream are freed.
- This function discards any unprocessed input and does not flush any
- pending output.
-
- deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the
- stream state was inconsistent, Z_DATA_ERROR if the stream was freed
- prematurely (some input or output was discarded). In the error case,
- msg may be set but then points to a static string (which must not be
- deallocated).
-*/
-
-
-/*
-ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm));
-
- Initializes the internal stream state for decompression. The fields
- next_in, avail_in, zalloc, zfree and opaque must be initialized before by
- the caller. If next_in is not Z_NULL and avail_in is large enough (the exact
- value depends on the compression method), inflateInit determines the
- compression method from the zlib header and allocates all data structures
- accordingly; otherwise the allocation will be deferred to the first call of
- inflate. If zalloc and zfree are set to Z_NULL, inflateInit updates them to
- use default allocation functions.
-
- inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
- memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
- version assumed by the caller. msg is set to null if there is no error
- message. inflateInit does not perform any decompression apart from reading
- the zlib header if present: this will be done by inflate(). (So next_in and
- avail_in may be modified, but next_out and avail_out are unchanged.)
-*/
-
-
-ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush));
-/*
- inflate decompresses as much data as possible, and stops when the input
- buffer becomes empty or the output buffer becomes full. It may introduce
- some output latency (reading input without producing any output) except when
- forced to flush.
-
- The detailed semantics are as follows. inflate performs one or both of the
- following actions:
-
- - Decompress more input starting at next_in and update next_in and avail_in
- accordingly. If not all input can be processed (because there is not
- enough room in the output buffer), next_in is updated and processing
- will resume at this point for the next call of inflate().
-
- - Provide more output starting at next_out and update next_out and avail_out
- accordingly. inflate() provides as much output as possible, until there
- is no more input data or no more space in the output buffer (see below
- about the flush parameter).
-
- Before the call of inflate(), the application should ensure that at least
- one of the actions is possible, by providing more input and/or consuming
- more output, and updating the next_* and avail_* values accordingly.
- The application can consume the uncompressed output when it wants, for
- example when the output buffer is full (avail_out == 0), or after each
- call of inflate(). If inflate returns Z_OK and with zero avail_out, it
- must be called again after making room in the output buffer because there
- might be more output pending.
-
- The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH,
- Z_FINISH, or Z_BLOCK. Z_SYNC_FLUSH requests that inflate() flush as much
- output as possible to the output buffer. Z_BLOCK requests that inflate() stop
- if and when it gets to the next deflate block boundary. When decoding the
- zlib or gzip format, this will cause inflate() to return immediately after
- the header and before the first block. When doing a raw inflate, inflate()
- will go ahead and process the first block, and will return when it gets to
- the end of that block, or when it runs out of data.
-
- The Z_BLOCK option assists in appending to or combining deflate streams.
- Also to assist in this, on return inflate() will set strm->data_type to the
- number of unused bits in the last byte taken from strm->next_in, plus 64
- if inflate() is currently decoding the last block in the deflate stream,
- plus 128 if inflate() returned immediately after decoding an end-of-block
- code or decoding the complete header up to just before the first byte of the
- deflate stream. The end-of-block will not be indicated until all of the
- uncompressed data from that block has been written to strm->next_out. The
- number of unused bits may in general be greater than seven, except when
- bit 7 of data_type is set, in which case the number of unused bits will be
- less than eight.
-
- inflate() should normally be called until it returns Z_STREAM_END or an
- error. However if all decompression is to be performed in a single step
- (a single call of inflate), the parameter flush should be set to
- Z_FINISH. In this case all pending input is processed and all pending
- output is flushed; avail_out must be large enough to hold all the
- uncompressed data. (The size of the uncompressed data may have been saved
- by the compressor for this purpose.) The next operation on this stream must
- be inflateEnd to deallocate the decompression state. The use of Z_FINISH
- is never required, but can be used to inform inflate that a faster approach
- may be used for the single inflate() call.
-
- In this implementation, inflate() always flushes as much output as
- possible to the output buffer, and always uses the faster approach on the
- first call. So the only effect of the flush parameter in this implementation
- is on the return value of inflate(), as noted below, or when it returns early
- because Z_BLOCK is used.
-
- If a preset dictionary is needed after this call (see inflateSetDictionary
- below), inflate sets strm->adler to the adler32 checksum of the dictionary
- chosen by the compressor and returns Z_NEED_DICT; otherwise it sets
- strm->adler to the adler32 checksum of all output produced so far (that is,
- total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described
- below. At the end of the stream, inflate() checks that its computed adler32
- checksum is equal to that saved by the compressor and returns Z_STREAM_END
- only if the checksum is correct.
-
- inflate() will decompress and check either zlib-wrapped or gzip-wrapped
- deflate data. The header type is detected automatically. Any information
- contained in the gzip header is not retained, so applications that need that
- information should instead use raw inflate, see inflateInit2() below, or
- inflateBack() and perform their own processing of the gzip header and
- trailer.
-
- inflate() returns Z_OK if some progress has been made (more input processed
- or more output produced), Z_STREAM_END if the end of the compressed data has
- been reached and all uncompressed output has been produced, Z_NEED_DICT if a
- preset dictionary is needed at this point, Z_DATA_ERROR if the input data was
- corrupted (input stream not conforming to the zlib format or incorrect check
- value), Z_STREAM_ERROR if the stream structure was inconsistent (for example
- if next_in or next_out was NULL), Z_MEM_ERROR if there was not enough memory,
- Z_BUF_ERROR if no progress is possible or if there was not enough room in the
- output buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and
- inflate() can be called again with more input and more output space to
- continue decompressing. If Z_DATA_ERROR is returned, the application may then
- call inflateSync() to look for a good compression block if a partial recovery
- of the data is desired.
-*/
-
-
-ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm));
-/*
- All dynamically allocated data structures for this stream are freed.
- This function discards any unprocessed input and does not flush any
- pending output.
-
- inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state
- was inconsistent. In the error case, msg may be set but then points to a
- static string (which must not be deallocated).
-*/
-
- /* Advanced functions */
-
-/*
- The following functions are needed only in some special applications.
-*/
-
-/*
-ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm,
- int level,
- int method,
- int windowBits,
- int memLevel,
- int strategy));
-
- This is another version of deflateInit with more compression options. The
- fields next_in, zalloc, zfree and opaque must be initialized before by
- the caller.
-
- The method parameter is the compression method. It must be Z_DEFLATED in
- this version of the library.
-
- The windowBits parameter is the base two logarithm of the window size
- (the size of the history buffer). It should be in the range 8..15 for this
- version of the library. Larger values of this parameter result in better
- compression at the expense of memory usage. The default value is 15 if
- deflateInit is used instead.
-
- windowBits can also be -8..-15 for raw deflate. In this case, -windowBits
- determines the window size. deflate() will then generate raw deflate data
- with no zlib header or trailer, and will not compute an adler32 check value.
-
- windowBits can also be greater than 15 for optional gzip encoding. Add
- 16 to windowBits to write a simple gzip header and trailer around the
- compressed data instead of a zlib wrapper. The gzip header will have no
- file name, no extra data, no comment, no modification time (set to zero),
- no header crc, and the operating system will be set to 255 (unknown). If a
- gzip stream is being written, strm->adler is a crc32 instead of an adler32.
-
- The memLevel parameter specifies how much memory should be allocated
- for the internal compression state. memLevel=1 uses minimum memory but
- is slow and reduces compression ratio; memLevel=9 uses maximum memory
- for optimal speed. The default value is 8. See zconf.h for total memory
- usage as a function of windowBits and memLevel.
-
- The strategy parameter is used to tune the compression algorithm. Use the
- value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a
- filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no
- string match), or Z_RLE to limit match distances to one (run-length
- encoding). Filtered data consists mostly of small values with a somewhat
- random distribution. In this case, the compression algorithm is tuned to
- compress them better. The effect of Z_FILTERED is to force more Huffman
- coding and less string matching; it is somewhat intermediate between
- Z_DEFAULT and Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as fast as
- Z_HUFFMAN_ONLY, but give better compression for PNG image data. The strategy
- parameter only affects the compression ratio but not the correctness of the
- compressed output even if it is not set appropriately. Z_FIXED prevents the
- use of dynamic Huffman codes, allowing for a simpler decoder for special
- applications.
-
- deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
- memory, Z_STREAM_ERROR if a parameter is invalid (such as an invalid
- method). msg is set to null if there is no error message. deflateInit2 does
- not perform any compression: this will be done by deflate().
-*/
-
-ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm,
- const Bytef *dictionary,
- uInt dictLength));
-/*
- Initializes the compression dictionary from the given byte sequence
- without producing any compressed output. This function must be called
- immediately after deflateInit, deflateInit2 or deflateReset, before any
- call of deflate. The compressor and decompressor must use exactly the same
- dictionary (see inflateSetDictionary).
-
- The dictionary should consist of strings (byte sequences) that are likely
- to be encountered later in the data to be compressed, with the most commonly
- used strings preferably put towards the end of the dictionary. Using a
- dictionary is most useful when the data to be compressed is short and can be
- predicted with good accuracy; the data can then be compressed better than
- with the default empty dictionary.
-
- Depending on the size of the compression data structures selected by
- deflateInit or deflateInit2, a part of the dictionary may in effect be
- discarded, for example if the dictionary is larger than the window size in
- deflate or deflate2. Thus the strings most likely to be useful should be
- put at the end of the dictionary, not at the front. In addition, the
- current implementation of deflate will use at most the window size minus
- 262 bytes of the provided dictionary.
-
- Upon return of this function, strm->adler is set to the adler32 value
- of the dictionary; the decompressor may later use this value to determine
- which dictionary has been used by the compressor. (The adler32 value
- applies to the whole dictionary even if only a subset of the dictionary is
- actually used by the compressor.) If a raw deflate was requested, then the
- adler32 value is not computed and strm->adler is not set.
-
- deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a
- parameter is invalid (such as NULL dictionary) or the stream state is
- inconsistent (for example if deflate has already been called for this stream
- or if the compression method is bsort). deflateSetDictionary does not
- perform any compression: this will be done by deflate().
-*/
-
-ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest,
- z_streamp source));
-/*
- Sets the destination stream as a complete copy of the source stream.
-
- This function can be useful when several compression strategies will be
- tried, for example when there are several ways of pre-processing the input
- data with a filter. The streams that will be discarded should then be freed
- by calling deflateEnd. Note that deflateCopy duplicates the internal
- compression state which can be quite large, so this strategy is slow and
- can consume lots of memory.
-
- deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
- enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
- (such as zalloc being NULL). msg is left unchanged in both source and
- destination.
-*/
-
-ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm));
-/*
- This function is equivalent to deflateEnd followed by deflateInit,
- but does not free and reallocate all the internal compression state.
- The stream will keep the same compression level and any other attributes
- that may have been set by deflateInit2.
-
- deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent (such as zalloc or state being NULL).
-*/
-
-ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm,
- int level,
- int strategy));
-/*
- Dynamically update the compression level and compression strategy. The
- interpretation of level and strategy is as in deflateInit2. This can be
- used to switch between compression and straight copy of the input data, or
- to switch to a different kind of input data requiring a different
- strategy. If the compression level is changed, the input available so far
- is compressed with the old level (and may be flushed); the new level will
- take effect only at the next call of deflate().
-
- Before the call of deflateParams, the stream state must be set as for
- a call of deflate(), since the currently available input may have to
- be compressed and flushed. In particular, strm->avail_out must be non-zero.
-
- deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source
- stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR
- if strm->avail_out was zero.
-*/
-
-ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm,
- int good_length,
- int max_lazy,
- int nice_length,
- int max_chain));
-/*
- Fine tune deflate's internal compression parameters. This should only be
- used by someone who understands the algorithm used by zlib's deflate for
- searching for the best matching string, and even then only by the most
- fanatic optimizer trying to squeeze out the last compressed bit for their
- specific input data. Read the deflate.c source code for the meaning of the
- max_lazy, good_length, nice_length, and max_chain parameters.
-
- deflateTune() can be called after deflateInit() or deflateInit2(), and
- returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream.
- */
-
-ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm,
- uLong sourceLen));
-/*
- deflateBound() returns an upper bound on the compressed size after
- deflation of sourceLen bytes. It must be called after deflateInit()
- or deflateInit2(). This would be used to allocate an output buffer
- for deflation in a single pass, and so would be called before deflate().
-*/
-
-ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm,
- int bits,
- int value));
-/*
- deflatePrime() inserts bits in the deflate output stream. The intent
- is that this function is used to start off the deflate output with the
- bits leftover from a previous deflate stream when appending to it. As such,
- this function can only be used for raw deflate, and must be used before the
- first deflate() call after a deflateInit2() or deflateReset(). bits must be
- less than or equal to 16, and that many of the least significant bits of
- value will be inserted in the output.
-
- deflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent.
-*/
-
-ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm,
- gz_headerp head));
-/*
- deflateSetHeader() provides gzip header information for when a gzip
- stream is requested by deflateInit2(). deflateSetHeader() may be called
- after deflateInit2() or deflateReset() and before the first call of
- deflate(). The text, time, os, extra field, name, and comment information
- in the provided gz_header structure are written to the gzip header (xflag is
- ignored -- the extra flags are set according to the compression level). The
- caller must assure that, if not Z_NULL, name and comment are terminated with
- a zero byte, and that if extra is not Z_NULL, that extra_len bytes are
- available there. If hcrc is true, a gzip header crc is included. Note that
- the current versions of the command-line version of gzip (up through version
- 1.3.x) do not support header crc's, and will report that it is a "multi-part
- gzip file" and give up.
-
- If deflateSetHeader is not used, the default gzip header has text false,
- the time set to zero, and os set to 255, with no extra, name, or comment
- fields. The gzip header is returned to the default state by deflateReset().
-
- deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent.
-*/
-
-/*
-ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
- int windowBits));
-
- This is another version of inflateInit with an extra parameter. The
- fields next_in, avail_in, zalloc, zfree and opaque must be initialized
- before by the caller.
-
- The windowBits parameter is the base two logarithm of the maximum window
- size (the size of the history buffer). It should be in the range 8..15 for
- this version of the library. The default value is 15 if inflateInit is used
- instead. windowBits must be greater than or equal to the windowBits value
- provided to deflateInit2() while compressing, or it must be equal to 15 if
- deflateInit2() was not used. If a compressed stream with a larger window
- size is given as input, inflate() will return with the error code
- Z_DATA_ERROR instead of trying to allocate a larger window.
-
- windowBits can also be -8..-15 for raw inflate. In this case, -windowBits
- determines the window size. inflate() will then process raw deflate data,
- not looking for a zlib or gzip header, not generating a check value, and not
- looking for any check values for comparison at the end of the stream. This
- is for use with other formats that use the deflate compressed data format
- such as zip. Those formats provide their own check values. If a custom
- format is developed using the raw deflate format for compressed data, it is
- recommended that a check value such as an adler32 or a crc32 be applied to
- the uncompressed data as is done in the zlib, gzip, and zip formats. For
- most applications, the zlib format should be used as is. Note that comments
- above on the use in deflateInit2() applies to the magnitude of windowBits.
-
- windowBits can also be greater than 15 for optional gzip decoding. Add
- 32 to windowBits to enable zlib and gzip decoding with automatic header
- detection, or add 16 to decode only the gzip format (the zlib format will
- return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is
- a crc32 instead of an adler32.
-
- inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
- memory, Z_STREAM_ERROR if a parameter is invalid (such as a null strm). msg
- is set to null if there is no error message. inflateInit2 does not perform
- any decompression apart from reading the zlib header if present: this will
- be done by inflate(). (So next_in and avail_in may be modified, but next_out
- and avail_out are unchanged.)
-*/
-
-ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm,
- const Bytef *dictionary,
- uInt dictLength));
-/*
- Initializes the decompression dictionary from the given uncompressed byte
- sequence. This function must be called immediately after a call of inflate,
- if that call returned Z_NEED_DICT. The dictionary chosen by the compressor
- can be determined from the adler32 value returned by that call of inflate.
- The compressor and decompressor must use exactly the same dictionary (see
- deflateSetDictionary). For raw inflate, this function can be called
- immediately after inflateInit2() or inflateReset() and before any call of
- inflate() to set the dictionary. The application must insure that the
- dictionary that was used for compression is provided.
-
- inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a
- parameter is invalid (such as NULL dictionary) or the stream state is
- inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the
- expected one (incorrect adler32 value). inflateSetDictionary does not
- perform any decompression: this will be done by subsequent calls of
- inflate().
-*/
-
-ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm));
-/*
- Skips invalid compressed data until a full flush point (see above the
- description of deflate with Z_FULL_FLUSH) can be found, or until all
- available input is skipped. No output is provided.
-
- inflateSync returns Z_OK if a full flush point has been found, Z_BUF_ERROR
- if no more input was provided, Z_DATA_ERROR if no flush point has been found,
- or Z_STREAM_ERROR if the stream structure was inconsistent. In the success
- case, the application may save the current current value of total_in which
- indicates where valid compressed data was found. In the error case, the
- application may repeatedly call inflateSync, providing more input each time,
- until success or end of the input data.
-*/
-
-ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest,
- z_streamp source));
-/*
- Sets the destination stream as a complete copy of the source stream.
-
- This function can be useful when randomly accessing a large stream. The
- first pass through the stream can periodically record the inflate state,
- allowing restarting inflate at those points when randomly accessing the
- stream.
-
- inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
- enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
- (such as zalloc being NULL). msg is left unchanged in both source and
- destination.
-*/
-
-ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm));
-/*
- This function is equivalent to inflateEnd followed by inflateInit,
- but does not free and reallocate all the internal decompression state.
- The stream will keep attributes that may have been set by inflateInit2.
-
- inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent (such as zalloc or state being NULL).
-*/
-
-ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm,
- int bits,
- int value));
-/*
- This function inserts bits in the inflate input stream. The intent is
- that this function is used to start inflating at a bit position in the
- middle of a byte. The provided bits will be used before any bytes are used
- from next_in. This function should only be used with raw inflate, and
- should be used before the first inflate() call after inflateInit2() or
- inflateReset(). bits must be less than or equal to 16, and that many of the
- least significant bits of value will be inserted in the input.
-
- inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent.
-*/
-
-ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm,
- gz_headerp head));
-/*
- inflateGetHeader() requests that gzip header information be stored in the
- provided gz_header structure. inflateGetHeader() may be called after
- inflateInit2() or inflateReset(), and before the first call of inflate().
- As inflate() processes the gzip stream, head->done is zero until the header
- is completed, at which time head->done is set to one. If a zlib stream is
- being decoded, then head->done is set to -1 to indicate that there will be
- no gzip header information forthcoming. Note that Z_BLOCK can be used to
- force inflate() to return immediately after header processing is complete
- and before any actual data is decompressed.
-
- The text, time, xflags, and os fields are filled in with the gzip header
- contents. hcrc is set to true if there is a header CRC. (The header CRC
- was valid if done is set to one.) If extra is not Z_NULL, then extra_max
- contains the maximum number of bytes to write to extra. Once done is true,
- extra_len contains the actual extra field length, and extra contains the
- extra field, or that field truncated if extra_max is less than extra_len.
- If name is not Z_NULL, then up to name_max characters are written there,
- terminated with a zero unless the length is greater than name_max. If
- comment is not Z_NULL, then up to comm_max characters are written there,
- terminated with a zero unless the length is greater than comm_max. When
- any of extra, name, or comment are not Z_NULL and the respective field is
- not present in the header, then that field is set to Z_NULL to signal its
- absence. This allows the use of deflateSetHeader() with the returned
- structure to duplicate the header. However if those fields are set to
- allocated memory, then the application will need to save those pointers
- elsewhere so that they can be eventually freed.
-
- If inflateGetHeader is not used, then the header information is simply
- discarded. The header is always checked for validity, including the header
- CRC if present. inflateReset() will reset the process to discard the header
- information. The application would need to call inflateGetHeader() again to
- retrieve the header from the next gzip stream.
-
- inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
- stream state was inconsistent.
-*/
-
-/*
-ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits,
- unsigned char FAR *window));
-
- Initialize the internal stream state for decompression using inflateBack()
- calls. The fields zalloc, zfree and opaque in strm must be initialized
- before the call. If zalloc and zfree are Z_NULL, then the default library-
- derived memory allocation routines are used. windowBits is the base two
- logarithm of the window size, in the range 8..15. window is a caller
- supplied buffer of that size. Except for special applications where it is
- assured that deflate was used with small window sizes, windowBits must be 15
- and a 32K byte window must be supplied to be able to decompress general
- deflate streams.
-
- See inflateBack() for the usage of these routines.
-
- inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of
- the paramaters are invalid, Z_MEM_ERROR if the internal state could not
- be allocated, or Z_VERSION_ERROR if the version of the library does not
- match the version of the header file.
-*/
-
-typedef unsigned (*in_func) OF((void FAR *, unsigned char FAR * FAR *));
-typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned));
-
-ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm,
- in_func in, void FAR *in_desc,
- out_func out, void FAR *out_desc));
-/*
- inflateBack() does a raw inflate with a single call using a call-back
- interface for input and output. This is more efficient than inflate() for
- file i/o applications in that it avoids copying between the output and the
- sliding window by simply making the window itself the output buffer. This
- function trusts the application to not change the output buffer passed by
- the output function, at least until inflateBack() returns.
-
- inflateBackInit() must be called first to allocate the internal state
- and to initialize the state with the user-provided window buffer.
- inflateBack() may then be used multiple times to inflate a complete, raw
- deflate stream with each call. inflateBackEnd() is then called to free
- the allocated state.
-
- A raw deflate stream is one with no zlib or gzip header or trailer.
- This routine would normally be used in a utility that reads zip or gzip
- files and writes out uncompressed files. The utility would decode the
- header and process the trailer on its own, hence this routine expects
- only the raw deflate stream to decompress. This is different from the
- normal behavior of inflate(), which expects either a zlib or gzip header and
- trailer around the deflate stream.
-
- inflateBack() uses two subroutines supplied by the caller that are then
- called by inflateBack() for input and output. inflateBack() calls those
- routines until it reads a complete deflate stream and writes out all of the
- uncompressed data, or until it encounters an error. The function's
- parameters and return types are defined above in the in_func and out_func
- typedefs. inflateBack() will call in(in_desc, &buf) which should return the
- number of bytes of provided input, and a pointer to that input in buf. If
- there is no input available, in() must return zero--buf is ignored in that
- case--and inflateBack() will return a buffer error. inflateBack() will call
- out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. out()
- should return zero on success, or non-zero on failure. If out() returns
- non-zero, inflateBack() will return with an error. Neither in() nor out()
- are permitted to change the contents of the window provided to
- inflateBackInit(), which is also the buffer that out() uses to write from.
- The length written by out() will be at most the window size. Any non-zero
- amount of input may be provided by in().
-
- For convenience, inflateBack() can be provided input on the first call by
- setting strm->next_in and strm->avail_in. If that input is exhausted, then
- in() will be called. Therefore strm->next_in must be initialized before
- calling inflateBack(). If strm->next_in is Z_NULL, then in() will be called
- immediately for input. If strm->next_in is not Z_NULL, then strm->avail_in
- must also be initialized, and then if strm->avail_in is not zero, input will
- initially be taken from strm->next_in[0 .. strm->avail_in - 1].
-
- The in_desc and out_desc parameters of inflateBack() is passed as the
- first parameter of in() and out() respectively when they are called. These
- descriptors can be optionally used to pass any information that the caller-
- supplied in() and out() functions need to do their job.
-
- On return, inflateBack() will set strm->next_in and strm->avail_in to
- pass back any unused input that was provided by the last in() call. The
- return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR
- if in() or out() returned an error, Z_DATA_ERROR if there was a format
- error in the deflate stream (in which case strm->msg is set to indicate the
- nature of the error), or Z_STREAM_ERROR if the stream was not properly
- initialized. In the case of Z_BUF_ERROR, an input or output error can be
- distinguished using strm->next_in which will be Z_NULL only if in() returned
- an error. If strm->next is not Z_NULL, then the Z_BUF_ERROR was due to
- out() returning non-zero. (in() will always be called before out(), so
- strm->next_in is assured to be defined if out() returns non-zero.) Note
- that inflateBack() cannot return Z_OK.
-*/
-
-ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm));
-/*
- All memory allocated by inflateBackInit() is freed.
-
- inflateBackEnd() returns Z_OK on success, or Z_STREAM_ERROR if the stream
- state was inconsistent.
-*/
-
-ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void));
-/* Return flags indicating compile-time options.
-
- Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other:
- 1.0: size of uInt
- 3.2: size of uLong
- 5.4: size of voidpf (pointer)
- 7.6: size of z_off_t
-
- Compiler, assembler, and debug options:
- 8: DEBUG
- 9: ASMV or ASMINF -- use ASM code
- 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention
- 11: 0 (reserved)
-
- One-time table building (smaller code, but not thread-safe if true):
- 12: BUILDFIXED -- build static block decoding tables when needed
- 13: DYNAMIC_CRC_TABLE -- build CRC calculation tables when needed
- 14,15: 0 (reserved)
-
- Library content (indicates missing functionality):
- 16: NO_GZCOMPRESS -- gz* functions cannot compress (to avoid linking
- deflate code when not needed)
- 17: NO_GZIP -- deflate can't write gzip streams, and inflate can't detect
- and decode gzip streams (to avoid linking crc code)
- 18-19: 0 (reserved)
-
- Operation variations (changes in library functionality):
- 20: PKZIP_BUG_WORKAROUND -- slightly more permissive inflate
- 21: FASTEST -- deflate algorithm with only one, lowest compression level
- 22,23: 0 (reserved)
-
- The sprintf variant used by gzprintf (zero is best):
- 24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format
- 25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure!
- 26: 0 = returns value, 1 = void -- 1 means inferred string length returned
-
- Remainder:
- 27-31: 0 (reserved)
- */
-
-
- /* utility functions */
-
-/*
- The following utility functions are implemented on top of the
- basic stream-oriented functions. To simplify the interface, some
- default options are assumed (compression level and memory usage,
- standard memory allocation functions). The source code of these
- utility functions can easily be modified if you need special options.
-*/
-
-ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen,
- const Bytef *source, uLong sourceLen));
-/*
- Compresses the source buffer into the destination buffer. sourceLen is
- the byte length of the source buffer. Upon entry, destLen is the total
- size of the destination buffer, which must be at least the value returned
- by compressBound(sourceLen). Upon exit, destLen is the actual size of the
- compressed buffer.
- This function can be used to compress a whole file at once if the
- input file is mmap'ed.
- compress returns Z_OK if success, Z_MEM_ERROR if there was not
- enough memory, Z_BUF_ERROR if there was not enough room in the output
- buffer.
-*/
-
-ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen,
- const Bytef *source, uLong sourceLen,
- int level));
-/*
- Compresses the source buffer into the destination buffer. The level
- parameter has the same meaning as in deflateInit. sourceLen is the byte
- length of the source buffer. Upon entry, destLen is the total size of the
- destination buffer, which must be at least the value returned by
- compressBound(sourceLen). Upon exit, destLen is the actual size of the
- compressed buffer.
-
- compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
- memory, Z_BUF_ERROR if there was not enough room in the output buffer,
- Z_STREAM_ERROR if the level parameter is invalid.
-*/
-
-ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen));
-/*
- compressBound() returns an upper bound on the compressed size after
- compress() or compress2() on sourceLen bytes. It would be used before
- a compress() or compress2() call to allocate the destination buffer.
-*/
-
-ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen,
- const Bytef *source, uLong sourceLen));
-/*
- Decompresses the source buffer into the destination buffer. sourceLen is
- the byte length of the source buffer. Upon entry, destLen is the total
- size of the destination buffer, which must be large enough to hold the
- entire uncompressed data. (The size of the uncompressed data must have
- been saved previously by the compressor and transmitted to the decompressor
- by some mechanism outside the scope of this compression library.)
- Upon exit, destLen is the actual size of the compressed buffer.
- This function can be used to decompress a whole file at once if the
- input file is mmap'ed.
-
- uncompress returns Z_OK if success, Z_MEM_ERROR if there was not
- enough memory, Z_BUF_ERROR if there was not enough room in the output
- buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete.
-*/
-
-
-typedef voidp gzFile;
-
-ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode));
-/*
- Opens a gzip (.gz) file for reading or writing. The mode parameter
- is as in fopen ("rb" or "wb") but can also include a compression level
- ("wb9") or a strategy: 'f' for filtered data as in "wb6f", 'h' for
- Huffman only compression as in "wb1h", or 'R' for run-length encoding
- as in "wb1R". (See the description of deflateInit2 for more information
- about the strategy parameter.)
-
- gzopen can be used to read a file which is not in gzip format; in this
- case gzread will directly read from the file without decompression.
-
- gzopen returns NULL if the file could not be opened or if there was
- insufficient memory to allocate the (de)compression state; errno
- can be checked to distinguish the two cases (if errno is zero, the
- zlib error is Z_MEM_ERROR). */
-
-ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode));
-/*
- gzdopen() associates a gzFile with the file descriptor fd. File
- descriptors are obtained from calls like open, dup, creat, pipe or
- fileno (in the file has been previously opened with fopen).
- The mode parameter is as in gzopen.
- The next call of gzclose on the returned gzFile will also close the
- file descriptor fd, just like fclose(fdopen(fd), mode) closes the file
- descriptor fd. If you want to keep fd open, use gzdopen(dup(fd), mode).
- gzdopen returns NULL if there was insufficient memory to allocate
- the (de)compression state.
-*/
-
-ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy));
-/*
- Dynamically update the compression level or strategy. See the description
- of deflateInit2 for the meaning of these parameters.
- gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not
- opened for writing.
-*/
-
-ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len));
-/*
- Reads the given number of uncompressed bytes from the compressed file.
- If the input file was not in gzip format, gzread copies the given number
- of bytes into the buffer.
- gzread returns the number of uncompressed bytes actually read (0 for
- end of file, -1 for error). */
-
-ZEXTERN int ZEXPORT gzwrite OF((gzFile file,
- voidpc buf, unsigned len));
-/*
- Writes the given number of uncompressed bytes into the compressed file.
- gzwrite returns the number of uncompressed bytes actually written
- (0 in case of error).
-*/
-
-ZEXTERN int ZEXPORTVA gzprintf OF((gzFile file, const char *format, ...));
-/*
- Converts, formats, and writes the args to the compressed file under
- control of the format string, as in fprintf. gzprintf returns the number of
- uncompressed bytes actually written (0 in case of error). The number of
- uncompressed bytes written is limited to 4095. The caller should assure that
- this limit is not exceeded. If it is exceeded, then gzprintf() will return
- return an error (0) with nothing written. In this case, there may also be a
- buffer overflow with unpredictable consequences, which is possible only if
- zlib was compiled with the insecure functions sprintf() or vsprintf()
- because the secure snprintf() or vsnprintf() functions were not available.
-*/
-
-ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s));
-/*
- Writes the given null-terminated string to the compressed file, excluding
- the terminating null character.
- gzputs returns the number of characters written, or -1 in case of error.
-*/
-
-ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len));
-/*
- Reads bytes from the compressed file until len-1 characters are read, or
- a newline character is read and transferred to buf, or an end-of-file
- condition is encountered. The string is then terminated with a null
- character.
- gzgets returns buf, or Z_NULL in case of error.
-*/
-
-ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c));
-/*
- Writes c, converted to an unsigned char, into the compressed file.
- gzputc returns the value that was written, or -1 in case of error.
-*/
-
-ZEXTERN int ZEXPORT gzgetc OF((gzFile file));
-/*
- Reads one byte from the compressed file. gzgetc returns this byte
- or -1 in case of end of file or error.
-*/
-
-ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file));
-/*
- Push one character back onto the stream to be read again later.
- Only one character of push-back is allowed. gzungetc() returns the
- character pushed, or -1 on failure. gzungetc() will fail if a
- character has been pushed but not read yet, or if c is -1. The pushed
- character will be discarded if the stream is repositioned with gzseek()
- or gzrewind().
-*/
-
-ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush));
-/*
- Flushes all pending output into the compressed file. The parameter
- flush is as in the deflate() function. The return value is the zlib
- error number (see function gzerror below). gzflush returns Z_OK if
- the flush parameter is Z_FINISH and all output could be flushed.
- gzflush should be called only when strictly necessary because it can
- degrade compression.
-*/
-
-ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file,
- z_off_t offset, int whence));
-/*
- Sets the starting position for the next gzread or gzwrite on the
- given compressed file. The offset represents a number of bytes in the
- uncompressed data stream. The whence parameter is defined as in lseek(2);
- the value SEEK_END is not supported.
- If the file is opened for reading, this function is emulated but can be
- extremely slow. If the file is opened for writing, only forward seeks are
- supported; gzseek then compresses a sequence of zeroes up to the new
- starting position.
-
- gzseek returns the resulting offset location as measured in bytes from
- the beginning of the uncompressed stream, or -1 in case of error, in
- particular if the file is opened for writing and the new starting position
- would be before the current position.
-*/
-
-ZEXTERN int ZEXPORT gzrewind OF((gzFile file));
-/*
- Rewinds the given file. This function is supported only for reading.
-
- gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET)
-*/
-
-ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file));
-/*
- Returns the starting position for the next gzread or gzwrite on the
- given compressed file. This position represents a number of bytes in the
- uncompressed data stream.
-
- gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR)
-*/
-
-ZEXTERN int ZEXPORT gzeof OF((gzFile file));
-/*
- Returns 1 when EOF has previously been detected reading the given
- input stream, otherwise zero.
-*/
-
-ZEXTERN int ZEXPORT gzdirect OF((gzFile file));
-/*
- Returns 1 if file is being read directly without decompression, otherwise
- zero.
-*/
-
-ZEXTERN int ZEXPORT gzclose OF((gzFile file));
-/*
- Flushes all pending output if necessary, closes the compressed file
- and deallocates all the (de)compression state. The return value is the zlib
- error number (see function gzerror below).
-*/
-
-ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum));
-/*
- Returns the error message for the last error which occurred on the
- given compressed file. errnum is set to zlib error number. If an
- error occurred in the file system and not in the compression library,
- errnum is set to Z_ERRNO and the application may consult errno
- to get the exact error code.
-*/
-
-ZEXTERN void ZEXPORT gzclearerr OF((gzFile file));
-/*
- Clears the error and end-of-file flags for file. This is analogous to the
- clearerr() function in stdio. This is useful for continuing to read a gzip
- file that is being written concurrently.
-*/
-
- /* checksum functions */
-
-/*
- These functions are not related to compression but are exported
- anyway because they might be useful in applications using the
- compression library.
-*/
-
-ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len));
-/*
- Update a running Adler-32 checksum with the bytes buf[0..len-1] and
- return the updated checksum. If buf is NULL, this function returns
- the required initial value for the checksum.
- An Adler-32 checksum is almost as reliable as a CRC32 but can be computed
- much faster. Usage example:
-
- uLong adler = adler32(0L, Z_NULL, 0);
-
- while (read_buffer(buffer, length) != EOF) {
- adler = adler32(adler, buffer, length);
- }
- if (adler != original_adler) error();
-*/
-
-ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2,
- z_off_t len2));
-/*
- Combine two Adler-32 checksums into one. For two sequences of bytes, seq1
- and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for
- each, adler1 and adler2. adler32_combine() returns the Adler-32 checksum of
- seq1 and seq2 concatenated, requiring only adler1, adler2, and len2.
-*/
-
-ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len));
-/*
- Update a running CRC-32 with the bytes buf[0..len-1] and return the
- updated CRC-32. If buf is NULL, this function returns the required initial
- value for the for the crc. Pre- and post-conditioning (one's complement) is
- performed within this function so it shouldn't be done by the application.
- Usage example:
-
- uLong crc = crc32(0L, Z_NULL, 0);
-
- while (read_buffer(buffer, length) != EOF) {
- crc = crc32(crc, buffer, length);
- }
- if (crc != original_crc) error();
-*/
-
-ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2));
-
-/*
- Combine two CRC-32 check values into one. For two sequences of bytes,
- seq1 and seq2 with lengths len1 and len2, CRC-32 check values were
- calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32
- check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and
- len2.
-*/
-
-
- /* various hacks, don't look :) */
-
-/* deflateInit and inflateInit are macros to allow checking the zlib version
- * and the compiler's view of z_stream:
- */
-ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level,
- const char *version, int stream_size));
-ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm,
- const char *version, int stream_size));
-ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int level, int method,
- int windowBits, int memLevel,
- int strategy, const char *version,
- int stream_size));
-ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits,
- const char *version, int stream_size));
-ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits,
- unsigned char FAR *window,
- const char *version,
- int stream_size));
-#define deflateInit(strm, level) \
- deflateInit_((strm), (level), ZLIB_VERSION, sizeof(z_stream))
-#define inflateInit(strm) \
- inflateInit_((strm), ZLIB_VERSION, sizeof(z_stream))
-#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \
- deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\
- (strategy), ZLIB_VERSION, sizeof(z_stream))
-#define inflateInit2(strm, windowBits) \
- inflateInit2_((strm), (windowBits), ZLIB_VERSION, sizeof(z_stream))
-#define inflateBackInit(strm, windowBits, window) \
- inflateBackInit_((strm), (windowBits), (window), \
- ZLIB_VERSION, sizeof(z_stream))
-
-
-#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL)
- struct internal_state {int dummy;}; /* hack for buggy compilers */
-#endif
-
-ZEXTERN const char * ZEXPORT zError OF((int));
-ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp z));
-ZEXTERN const uLongf * ZEXPORT get_crc_table OF((void));
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* ZLIB_H */
package_list = ['pysam',
'pysam.include',
'pysam.include.samtools',
- 'pysam.include.bcftools',
- 'pysam.include.samtools.win32']
+ 'pysam.include.bcftools']
package_dirs = {'pysam': 'pysam',
'pysam.include.samtools': 'samtools',
'pysam.include.bcftools': 'bcftools'}
key, value = re.match(
"#define (\S+)\s+(\S+)", line).groups()
config_values[key] = value
- for key in ["ENABLE_PLUGINS",
+ for key in ["ENABLE_GCS",
+ "ENABLE_PLUGINS",
+ "ENABLE_S3",
"HAVE_COMMONCRYPTO",
- "HAVE_GMTIME_R",
"HAVE_HMAC",
- "HAVE_IRODS",
+ "HAVE_LIBBZ2",
"HAVE_LIBCURL",
+ "HAVE_LIBDEFLATE",
+ "HAVE_LIBLZMA",
"HAVE_MMAP"]:
outf.write("{} = {}\n".format(key, config_values[key]))
print ("# pysam: config_option: {}={}".format(key, config_values[key]))
File ex1.fa contains two sequences cut from the human genome
-build36. They were exatracted with command:
+build36. They were extracted with command:
samtools faidx human_b36.fa 2:2043966-2045540 20:67967-69550
Sequence names were changed manually for simplicity. File ex1.sam.gz
-contains MAQ alignments exatracted with:
+contains MAQ alignments extracted with:
(samtools view NA18507_maq.bam 2:2044001-2045500;
samtools view NA18507_maq.bam 20:68001-69500)
and processed with `samtools fixmate' to make it self-consistent as a
standalone alignment.
-To try samtools, you may run the following commands:
-
- samtools faidx ex1.fa # index the reference FASTA
- samtools import ex1.fa.fai ex1.sam.gz ex1.bam # SAM->BAM
- samtools index ex1.bam # index BAM
- samtools tview ex1.bam ex1.fa # view alignment
- samtools pileup -cf ex1.fa ex1.bam # pileup and consensus
- samtools pileup -cf ex1.fa -t ex1.fa.fai ex1.sam.gz
-
In order for the script pysam_test.py to work, you will need pysam
in your PYTHONPATH.
def testPresentOptionalFields(self):
self.assertEqual(
- self.reads[0].opt('NM'), 1,
+ self.reads[0].opt('NM'), 22,
"optional field mismatch in read 1, NM: %s != %s" %
- (self.reads[0].opt('NM'), 1))
+ (self.reads[0].opt('NM'), 22))
self.assertEqual(
self.reads[0].opt('RG'), 'L1',
"optional field mismatch in read 1, RG: %s != %s" %
def testTags(self):
self.assertEqual(self.reads[0].tags,
- [('NM', 1), ('RG', 'L1'),
+ [('NM', 22), ('RG', 'L1'),
('PG', 'P1'), ('XT', 'U')])
self.assertEqual(self.reads[1].tags,
[('MF', 18), ('RG', 'L2'),
os.unlink(tmpfilename)
-class TestDeNovoConstructionUserTags(TestDeNovoConstruction):
-
- '''test de novo construction with a header that contains lower-case tags.'''
-
- header = {'HD': {'VN': '1.0'},
- 'SQ': [{'LN': 1575, 'SN': 'chr1'},
- {'LN': 1584, 'SN': 'chr2'}],
- 'x1': {'A': 2, 'B': 5},
- 'x3': {'A': 6, 'B': 5},
- 'x2': {'A': 4, 'B': 5}}
-
- bamfile = os.path.join(BAM_DATADIR, "example_user_header.bam")
- samfile = os.path.join(BAM_DATADIR, "example_user_header.sam")
-
-
class TestEmptyHeader(unittest.TestCase):
'''see issue 84.'''
'ID': 'bwa',
'VN': '0.7.9a-r786',
'CL': 'bwa mem -p -t 8 -M -R '
- '@RG\tID:None\tSM:None\t/mnt/data/hg19.fa\t'
+ '@RG ID:None SM:None /mnt/data/hg19.fa '
'/mnt/analysis/default-0.fastq'}]})
cols = bam.pileup()
self.assertTrue(True)
+ def tearDown(self):
+ if os.path.exists('log.txt'):
+ os.unlink('log.txt')
+
def testFail1(self):
self.check(os.path.join(BAM_DATADIR, "ex9_fail.bam"),
False)
'''see issue 110'''
bamfile = "http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/phase3_EX_or_LC_only_alignment/data/HG00104/alignment/HG00104.chrom11.ILLUMINA.bwa.GBR.low_coverage.20130415.bam" # noqa
+ bambase = "HG00104.chrom11.ILLUMINA.bwa.GBR.low_coverage.20130415.bam" # noqa
def testRead(self):
data = f.header.copy()
self.assertTrue(data)
+ def tearDown(self):
+ if os.path.exists(self.bambase + ".bai"):
+ os.unlink(self.bambase + ".bai")
+
class TestLargeCigar(unittest.TestCase):
import pysam
+import pytest
import unittest
import os
import gzip
self.assertEqual(str(fastx_record), ">name\nsequence")
+class TestFastqProxy(unittest.TestCase):
+
+ def test_fastq_proxy_instantiation_raises_error(self):
+ self.assertRaises(ValueError,
+ pysam.FastqProxy)
+
+
if __name__ == "__main__":
unittest.main()
BAI=$(BAM:%.bam=%.bam.bai)
CRAM=ex1.cram ex2.cram ex3.cram
CRAI=$(CRAM:%.cram=%.cram.crai)
+NO_PG:=$(findstring --no-PG,$(shell samtools view))
# ex2.bam - bam file without index
# ex2.sam - as ex1.sam, but with header
ex2.sam.gz: ex1.bam ex1.bam.bai
- samtools view -h ex1.bam | gzip > ex2.sam.gz
+ samtools view $(NO_PG) -h ex1.bam | gzip > ex2.sam.gz
with_md.sam.gz: ex2.bam ex1.fa
- samtools calmd --output-fmt BAM $^ > $@
+ samtools calmd $(NO_PG) --output-fmt BAM $^ > $@
#%.bam: %.sam ex1.fa.fai
-# samtools import ex1.fa.fai $< $@
+# samtools view $(NO_PG) -bo $@ -t ex1.fa.fai $<
uncompressed.bam: ex2.sam
- samtools view -buS $< > $@
+ samtools view $(NO_PG) -buS $< > $@
%.bam: %.sam
- samtools view -bS $< > $@
+ samtools view $(NO_PG) -bS $< > $@
%.cram: %.sam
- samtools view -bC -T ex1.fa $< > $@
+ samtools view $(NO_PG) -bC -T ex1.fa $< > $@
%.cram.crai: %.cram
samtools index $<
samtools faidx ex1.fa
ex1.bam:ex1.sam.gz ex1.fa.fai
- samtools import ex1.fa.fai ex1.sam.gz ex1.bam
+ samtools view $(NO_PG) -bo ex1.bam -t ex1.fa.fai ex1.sam.gz
%.bam.bai:%.bam
samtools index $<
samtools index -c ex1_csi.bam
empty.bam: ex2.sam
- grep "^@" $< | samtools view -Sb - > $@
+ grep "^@" $< | samtools view $(NO_PG) -Sb - > $@
example_unmapped_reads_no_sq.bam: example_unmapped_reads_no_sq.sam
touch tmp.list
- samtools import tmp.list $< $@
+ samtools view $(NO_PG) -bo $@ -t tmp.list $<
rm -f tmp.list
example_bai.bam: ex1.bam
@PG ID:P2 VN:1.1
@CO this is a comment
@CO this is another comment
-read_28833_29006_6945 99 chr1 33 20 10M1D25M = 200 167 AGCTTAGCTAGCTACCTATATCTTGGTCTTGGCCG <<<<<<<<<<<<<<<<<<<<<:<9/,&,22;;<<< NM:i:1 RG:Z:L1 PG:Z:P1 XT:A:U
+read_28833_29006_6945 99 chr1 33 20 10M1D25M = 200 167 AGCTTAGCTAGCTACCTATATCTTGGTCTTGGCCG <<<<<<<<<<<<<<<<<<<<<:<9/,&,22;;<<< NM:i:22 RG:Z:L1 PG:Z:P1 XT:A:U
read_28701_28881_323b 147 chr2 88 30 35M = 500 412 ACCTATATCTTGGCCTTGGCCGATGCGGCCTTGCA <<<<<;<<<<7;:<<<6;<<<<<<<<<<<<7<<<< MF:i:18 RG:Z:L2 PG:Z:P2 XT:A:R
read_28701_28881_323c 147 chr2 88 30 35M = 500 412 ACCTATATCTTGGCCTTGGCCGATGCGGCCTTGCA <<<<<;<<<<7;:<<<6;<<<<<<<<<<<<7<<<<
-test_clipped1 99 chr2 997 20 4S6M1D20M5S = 200 167 AGCTTAGCTAGCTACCTATATCTTGGTCTTGGCCG <<<<<<<<<<<<<<<<<<<<<:<9/,&,22;;<<< NM:i:1 RG:Z:L1 PG:Z:P1 XT:A:U
+test_clipped1 99 chr2 997 20 4S6M1D20M5S = 200 167 AGCTTAGCTAGCTACCTATATCTTGGTCTTGGCCG <<<<<<<<<<<<<<<<<<<<<:<9/,&,22;;<<< NM:i:21 RG:Z:L1 PG:Z:P1 XT:A:U
-@HD VN:1.3 SO:coordinate
+@HD VN:1.3 SO:coordinate
+++ /dev/null
-@HD VN:1.0
-@SQ SN:chr1 LN:1575
-@SQ SN:chr2 LN:1584
-@x1 A:2 B:5
-@x2 A:4 B:5
-@x3 A:6 B:5
-read_28833_29006_6945 99 chr1 33 20 10M1D25M = 200 167 AGCTTAGCTAGCTACCTATATCTTGGTCTTGGCCG <<<<<<<<<<<<<<<<<<<<<:<9/,&,22;;<<< NM:i:1 RG:Z:L1
-read_28701_28881_323b 147 chr2 88 30 35M = 500 412 ACCTATATCTTGGCCTTGGCCGATGCGGCCTTGCA <<<<<;<<<<7;:<<<6;<<<<<<<<<<<<7<<<< MF:i:18 RG:Z:L2
@SQ SN:chr1 LN:1575
@SQ SN:chr2 LN:1584
-@PG ID:bwa PN:bwa VN:0.7.9a-r786 CL:bwa mem -p -t 8 -M -R @RG ID:None SM:None /mnt/data/hg19.fa /mnt/analysis/default-0.fastq
+@PG ID:bwa PN:bwa VN:0.7.9a-r786 CL:bwa mem -p -t 8 -M -R @RG ID:None SM:None /mnt/data/hg19.fa /mnt/analysis/default-0.fastq
EAS56_57:6:190:289:82 69 chr1 100 0 * = 100 0 CTCAAGGTTGTTGCAAGGGGGTCTATGTGAACAAA <<<7<<<;<<<<<<<<8;;<7;4<;<;;;;;94<; MF:i:192
EAS56_57:6:190:289:82 137 chr1 100 73 35M = 100 0 AGGGGTGCAGAGCCGAGTCACGGGGTTGCCAGCAC <<<<<<;<<<<<<<<<<;<<;<<<<;8<6;9;;2; MF:i:64 Aq:i:0 NM:i:0 UQ:i:0 H0:i:1 H1:i:0
EAS51_64:3:190:727:308 99 chr1 103 99 35M = 263 195 GGTGCAGAGCCGAGTCACGGGGTTGCCAGCACAGG <<<<<<<<<<<<<<<<<<<<<<<<<<<::<<<844 MF:i:18 Aq:i:73 NM:i:0 UQ:i:0 H0:i:1 H1:i:0
"cat -o %(out)s_ex1.cat.bam ex1.bam ex1.bam",
"targetcut ex1.bam > %(out)s_ex1.targetcut",
"phase ex1.bam > %(out)s_ex1.phase",
- "import ex1.fa.fai ex1.sam.gz %(out)s_ex1.bam",
+ "view -bt ex1.fa.fai ex1.sam.gz > %(out)s_ex1.bam",
"bam2fq ex1.bam > %(out)s_ex1.bam2fq",
# TODO: not the same
# "pad2unpad -T ex1.fa ex2.bam > %(out)s_ex2.unpad",
]
map_command = {
- "import": "samimport"}
+ }
executable = "samtools"
# ]
# map_command = {
-# "import": "samimport"}
+# }
# executable = "bcftools"
import pysam
import os
+import pytest
from TestUtils import BAM_DATADIR
projects / python-pysam.git / commitdiff