From a350c3d5118b5f9e05b056d596bce7055f3037b3 Mon Sep 17 00:00:00 2001 From: caleblareau Date: Mon, 6 Jul 2026 10:04:15 -0400 Subject: [PATCH] Modernize packaging, add test suite, remove shell-string system calls Step 1 - Dependencies & packaging: - Replace setup.py/setup.cfg with PEP 621 pyproject.toml (fixes distutils import removed in Python 3.12+, dedupes biopython, drops py2 universal wheel, requires-python >=3.9, pytest moved to a [test] extra) - Replace deprecated pkg_resources with importlib.metadata (bap_version helper) Step 2 - Testing: - Add pytest suite: unit tests (test_helpers.py), hermetic CLI smoke tests (test_cli_smoke.py), and auto-skipping end-to-end tests (test_integration.py) - Add GitHub Actions CI across Python 3.9-3.12 - Rewrite tests/README.md into a real testing guide Step 3 - Refactor (no shell-string system calls) in active CLIs: - Add run_cmd() subprocess wrapper; replace os.system()/os.popen()/cp with subprocess arg-lists, glob, and shutil; launch child scripts via sys.executable - Migrate ruamel.yaml dumping to the modern YAML() API - Fix latent bugs: Python-2 itertools.imap in string_hamming_distance, abs_file->abs_path, missing import subprocess, undefined macs2_genome_size/ bs_genome, and import-time PyYAML dependency Legacy v1 CLI (cli_old_dontEdit.py) and Snakefile shell pipelines are intentionally left unchanged. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 33 +++++++ MANIFEST.in | 4 +- README.md | 22 +++++ bap/bap2ProjectClass.py | 8 +- bap/bapFragProjectClass.py | 6 +- bap/bapHelp.py | 61 ++++++++++-- bap/barcode/cli_barcode.py | 56 +++++------ bap/barcode/cli_scaleatac.py | 17 +--- bap/cli_bap2.py | 87 +++++++++-------- bap/cli_bap_bulk_frag.py | 62 ++++++------ bap/cli_bap_frag.py | 62 ++++++------ bap/cli_reanno.py | 14 +-- pyproject.toml | 74 ++++++++++++++ setup.cfg | 6 -- setup.py | 53 ---------- tests/README.md | 99 ++++++++++++++----- tests/conftest.py | 21 ++++ tests/test_cli_smoke.py | 45 +++++++++ tests/test_helpers.py | 184 +++++++++++++++++++++++++++++++++++ tests/test_integration.py | 42 ++++++++ 20 files changed, 698 insertions(+), 258 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100644 setup.py create mode 100644 tests/conftest.py create mode 100644 tests/test_cli_smoke.py create mode 100644 tests/test_helpers.py create mode 100644 tests/test_integration.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2fec743 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +name: CI + +on: + push: + branches: [master, main] + pull_request: + +jobs: + test: + name: Unit & smoke tests (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install package and test dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[test] + + - name: Run unit and smoke tests + # Integration tests require samtools/bedtools/R/snakemake and are + # skipped automatically when those tools are not installed. + run: pytest -m "not integration" -v diff --git a/MANIFEST.in b/MANIFEST.in index b129a5a..d6d0bbb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,5 +9,7 @@ recursive-include bap * # Misc include Authors.rst -include LICENSE.txt +include LICENSE include NEWS +include README.md +include pyproject.toml diff --git a/README.md b/README.md index d6728e2..95678ee 100644 --- a/README.md +++ b/README.md @@ -10,5 +10,27 @@ data files for downstream analyses for droplet-based single-cell ATAC-seq data. [See the bap wiki page](https://github.com/caleblareau/bap/wiki) for details and FAQs about bap. +## Installation + +``` +pip install bap-atac # from PyPI +pip install -e . # from a local checkout (development) +``` + +Requires Python >= 3.9. The pipeline also shells out to `samtools`, `bedtools`, +`R`, and `snakemake`, which must be available on your `PATH`. + +## Testing + +Automated tests use `pytest`. Install the test extras and run the fast suite: + +``` +pip install -e .[test] +pytest -m "not integration" # unit + CLI smoke tests, no external tools needed +pytest # also runs end-to-end tests when the external tools are present +``` + +See [`tests/README.md`](tests/README.md) for the full testing guide. + diff --git a/bap/bap2ProjectClass.py b/bap/bap2ProjectClass.py index fb3505f..7d85b11 100644 --- a/bap/bap2ProjectClass.py +++ b/bap/bap2ProjectClass.py @@ -8,8 +8,6 @@ import itertools import time import platform -from ruamel import yaml -from pkg_resources import get_distribution from .bapHelp import * def getBfiles(bedtools_genome, blacklist_file, reference_genome, script_dir, supported_genomes): @@ -75,7 +73,7 @@ def __init__(self, script_dir, supported_genomes, mode, input, output, name, nco #---------------------------------- # Assign straightforward attributes #---------------------------------- - self.bap_version = get_distribution('bap-atac').version + self.bap_version = bap_version() self.script_dir = script_dir self.mode = mode self.output = output @@ -173,12 +171,12 @@ def __init__(self, script_dir, supported_genomes, mode, input, output, name, nco else: click.echo(gettime() + "Could not identify this reference genome: %s" % self.reference_genome) click.echo(gettime() + "Attempting to infer necessary input files from user specification.") - necessary = [bedtools_genome, blacklist_file, tss_file, macs2_genome_size, bs_genome] + necessary = [bedtools_genome, blacklist_file, tss_file] if '' in necessary: if reference_genome == '': sys.exit("ERROR: specify valid reference genome with --reference-genome flag; QUITTING") else: - sys.exit("ERROR: non-supported reference genome specified so these five must be validly specified: --bedtools-genome, --blacklist-file, --tss-file; QUITTING") + sys.exit("ERROR: non-supported reference genome specified so these three must be validly specified: --bedtools-genome, --blacklist-file, --tss-file; QUITTING") if(reference_genome in ["hg19-mm10", "hg19_mm10_c"]): self.speciesMix = "yes" diff --git a/bap/bapFragProjectClass.py b/bap/bapFragProjectClass.py index 820742a..e7c0d3f 100644 --- a/bap/bapFragProjectClass.py +++ b/bap/bapFragProjectClass.py @@ -8,8 +8,6 @@ import itertools import time import platform -from ruamel import yaml -from pkg_resources import get_distribution from .bapHelp import * def getBfiles(bedtools_genome, blacklist_file, reference_genome, script_dir, supported_genomes): @@ -69,7 +67,7 @@ def __init__(self, script_dir, supported_genomes, input, output, name, ncores, r #---------------------------------- # Assign straightforward attributes #---------------------------------- - self.bap_version = get_distribution('bap-atac').version + self.bap_version = bap_version() self.script_dir = script_dir self.bamfile = input self.name = name @@ -202,7 +200,7 @@ def __init__(self, script_dir, supported_genomes, input, output, name, ncores, r #---------------------------------- # Assign straightforward attributes #---------------------------------- - self.bap_version = get_distribution('bap').version + self.bap_version = bap_version() self.script_dir = script_dir self.bamfile = input self.name = name diff --git a/bap/bapHelp.py b/bap/bapHelp.py index 2cbf79b..3a5f10c 100644 --- a/bap/bapHelp.py +++ b/bap/bapHelp.py @@ -1,4 +1,3 @@ -import itertools import time import shutil import re @@ -6,18 +5,61 @@ import sys import csv import gzip -from ruamel import yaml +import glob +import subprocess from functools import partial +from importlib.metadata import version, PackageNotFoundError + + +def bap_version(): + ''' + Return the installed bap-atac version, or "unknown" if the package + metadata cannot be located (e.g. running from an uninstalled checkout). + ''' + try: + return version("bap-atac") + except PackageNotFoundError: + return "unknown" + + +def run_cmd(args, log_file=None, log_stderr_only=False, shell=False, check=True): + ''' + Run an external command from a list of arguments (no shell by default), + raising a clear error if it fails. This replaces the historical + os.system() calls, avoiding shell-injection and quoting issues. + + args: list of command arguments (or a string when shell=True) + log_file: optional path; command output is written here. By default + both stdout and stderr are captured (equivalent to `&>`). + log_stderr_only: when True, only stderr is captured to log_file and stdout + is inherited by the terminal (equivalent to `2>`). + shell: only set True for commands that need a shell pipeline + check: when True, exit with a clear message on a non-zero return + code; set False to let the caller inspect the result + ''' + if log_file is not None: + with open(log_file, 'w') as fh: + if log_stderr_only: + completed = subprocess.run(args, stderr=fh, shell=shell) + else: + completed = subprocess.run(args, stdout=fh, stderr=subprocess.STDOUT, shell=shell) + else: + completed = subprocess.run(args, shell=shell) + if check and completed.returncode != 0: + printed = args if isinstance(args, str) else " ".join(str(a) for a in args) + sys.exit(gettime() + "ERROR: command failed (exit %d): %s" % (completed.returncode, printed)) + return completed + def string_hamming_distance(str1, str2): ''' Fast hamming distance over 2 strings known to be of same length. - In information theory, the Hamming distance between two strings of equal - length is the number of positions at which the corresponding symbols + In information theory, the Hamming distance between two strings of equal + length is the number of positions at which the corresponding symbols are different. eg "karolin" and "kathrin" is 3. ''' - return sum(itertools.imap(operator.ne, str1, str2)) + return sum(c1 != c2 for c1, c2 in zip(str1, str2)) def intersection(lst1, lst2): lst3 = [value for value in lst1 if value in lst2] @@ -67,7 +109,7 @@ def get_software_path(tool, abs_path): if(abs_path == ""): sys.exit("ERROR: cannot find "+tool+" in environment; add it to user PATH environment or specify executable using a flag.") if(abs_path != ""): - if(os.path.isfile(abs_file)): + if(os.path.isfile(abs_path)): tool_path = abs_path return(tool_path) @@ -96,7 +138,10 @@ def check_R_packages(required_packages, R_path): ''' Determines whether or not R packages are properly installed ''' - installed_packages = os.popen(R_path + ''' -e "installed.packages()" | awk '{print $1}' | sort | uniq''').read().strip().split("\n") + completed = subprocess.run( + [R_path, "--vanilla", "--slave", "-e", "cat(rownames(installed.packages()), sep='\\n')"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + installed_packages = completed.stdout.strip().split("\n") if(not set(required_packages) < set(installed_packages)): sys.exit("ERROR: cannot find the following R package: " + str(set(required_packages) - set(installed_packages)) + "\n" + "Install it in your R console and then try rerunning proatac (but there may be other missing dependencies).") @@ -150,7 +195,7 @@ def list_duplicates_of(seq,item): # Otherwise figure out .fastq files from directory, do the merging, and infer sample names else: - files = os.popen("ls " + input.rstrip("/") +"/*.fastq.gz").read().strip().split("\n") + files = sorted(glob.glob(input.rstrip("/") + "/*.fastq.gz")) if(len(files) < 2): sys.exit("ERROR: input determined to be a directory but no paired .fastq.gz files found; QUITTING") files1 = [filename.replace("_R1", "_1").replace("_R2", "_1").replace("_2", "_1") for filename in files] diff --git a/bap/barcode/cli_barcode.py b/bap/barcode/cli_barcode.py index 38e812f..c8e2018 100644 --- a/bap/barcode/cli_barcode.py +++ b/bap/barcode/cli_barcode.py @@ -2,22 +2,11 @@ import os import os.path import sys -import shutil -import yaml -import random -import string -import itertools -import time -import csv -import re -from itertools import groupby from ..bapHelp import * -from pkg_resources import get_distribution -from subprocess import call, check_call @click.command() -@click.version_option() +@click.version_option(version=bap_version()) @click.argument('mode', type=click.Choice(['v1.0', 'v2.0', 'v2.1', 'v2.1-multi', '10X-v1'])) @@ -42,36 +31,37 @@ def main(mode, fastq1, fastq2, fastqi, output, ncores, nreads, nmismatches, reve mode = ['v1.0', 'v2.0', 'v2.1', 'v2.1-multi', '10X-v1'] for bead design\n """ - __version__ = get_distribution('bap-atac').version + __version__ = bap_version() script_dir = os.path.dirname(os.path.realpath(__file__)) click.echo(gettime() + "Starting de-barcoding from bap pipeline v%s\n" % __version__) # Parse user settings - core_call1 = " --fastq1 " + fastq1 + " --fastq2 " + fastq2 + " --ncores " + str(ncores) - core_call2 = " --nreads " + str(nreads) + " --nmismatches " + str(nmismatches) + " --output " + output - core_call = core_call1 + core_call2 - + core_call = [ + "--fastq1", fastq1, "--fastq2", fastq2, "--ncores", str(ncores), + "--nreads", str(nreads), "--nmismatches", str(nmismatches), "--output", output, + ] + # Handle mode to handle the configuration and make the right system call if(mode == "v1.0"): - cmd = 'python '+script_dir+'/modes/biorad_v1.py ' - earlier = " --constant1 " + "TAGCCATCGCATTGC" + " --constant2 " + "TACCTCTGAGCTGAA" - later = " --nextera " + "TCGTCGGCAGCGTC" + " --me " + "AGATGTGTATAAGAGACAG" + script = script_dir + "/modes/biorad_v1.py" + earlier = ["--constant1", "TAGCCATCGCATTGC", "--constant2", "TACCTCTGAGCTGAA"] + later = ["--nextera", "TCGTCGGCAGCGTC", "--me", "AGATGTGTATAAGAGACAG"] elif(mode == "v2.0"): - cmd = 'python '+script_dir+'/modes/biorad_v2.py ' - earlier = " --constant1 " + "TATGCATGAC" + " --constant2 " + "AGTCACTGAG" - later = " --nextera " + "TGGTAGAGAGGGTG" + " --me " + "AGATGTGTATAAGAGACAG" + script = script_dir + "/modes/biorad_v2.py" + earlier = ["--constant1", "TATGCATGAC", "--constant2", "AGTCACTGAG"] + later = ["--nextera", "TGGTAGAGAGGGTG", "--me", "AGATGTGTATAAGAGACAG"] elif(mode == "v2.1"): - cmd = 'python '+script_dir+'/modes/biorad_v2.py ' - earlier = " --constant1 " + "TATGCATGAC" + " --constant2 " + "AGTCACTGAG" - later = " --nextera " + "TCGTCGGCAGCGTC" + " --me " + "AGATGTGTATAAGAGACAG" + script = script_dir + "/modes/biorad_v2.py" + earlier = ["--constant1", "TATGCATGAC", "--constant2", "AGTCACTGAG"] + later = ["--nextera", "TCGTCGGCAGCGTC", "--me", "AGATGTGTATAAGAGACAG"] elif(mode == "v2.1-multi"): - cmd = 'python '+script_dir+'/modes/biorad_v2-multi.py ' - earlier = " --constant1 " + "TATGCATGAC" + " --constant2 " + "AGTCACTGAG" - later = " --nextera " + "TCGTCGGCAGCGTC" + " --me " + "AGATGTGTATAAGAGACAG" + script = script_dir + "/modes/biorad_v2-multi.py" + earlier = ["--constant1", "TATGCATGAC", "--constant2", "AGTCACTGAG"] + later = ["--nextera", "TCGTCGGCAGCGTC", "--me", "AGATGTGTATAAGAGACAG"] else: sys.exit(gettime() + "User-supplied mode %s not found!" % mode) - - # Assemble the final call - sys_call = cmd + earlier + later + core_call + " 2> "+output+".stderr.txt" - os.system(sys_call) + + # Assemble the final call; stderr is captured to a log file (stdout stays on the terminal) + sys_call = [sys.executable, script] + earlier + later + core_call + run_cmd(sys_call, log_file=output + ".stderr.txt", log_stderr_only=True) diff --git a/bap/barcode/cli_scaleatac.py b/bap/barcode/cli_scaleatac.py index 52a3496..f3ffdc2 100644 --- a/bap/barcode/cli_scaleatac.py +++ b/bap/barcode/cli_scaleatac.py @@ -2,30 +2,19 @@ import os import os.path import sys -import shutil -import yaml -import random -import string -import itertools -import time import glob +import gzip -import csv -import re -from itertools import groupby from ..bapHelp import * from .scaleHelp import * -from pkg_resources import get_distribution -from subprocess import call, check_call - from multiprocessing import Pool, freeze_support from Bio import SeqIO from Bio.Seq import Seq from Bio.SeqIO.QualityIO import FastqGeneralIterator @click.command() -@click.version_option() +@click.version_option(version=bap_version()) @click.option('--fastqs', '-f', help='Path of folder created by mkfastq or bcl2fastq; can be comma separated that will be collapsed into one output.') @click.option('--sample', '-s', help='Prefix of the filenames of FASTQs to select; can be comma separated that will be collapsed into one output.') @@ -43,7 +32,7 @@ def main(fastqs, sample, output, ncores, nreads): Trims, processes Tn5 barcode, and corrects bead barcode in one shot \n """ - __version__ = get_distribution('bap-atac').version + __version__ = bap_version() script_dir = os.path.dirname(os.path.realpath(__file__)) click.echo(gettime() + "Starting de-barcoding of scale data from bap pipeline v%s" % __version__) diff --git a/bap/cli_bap2.py b/bap/cli_bap2.py index c16b19d..7adaca9 100644 --- a/bap/cli_bap2.py +++ b/bap/cli_bap2.py @@ -3,22 +3,15 @@ import os.path import sys import shutil -import yaml -import random -import string -import itertools -import time +import glob import pysam -from pkg_resources import get_distribution -from subprocess import call, check_call from .bapHelp import * from .bap2ProjectClass import * -from ruamel import yaml -from ruamel.yaml.scalarstring import SingleQuotedScalarString as sqs +from ruamel.yaml import YAML @click.command() -@click.version_option() +@click.version_option(version=bap_version()) @click.argument('mode', type=click.Choice(['bam', 'check', 'support'])) @@ -86,15 +79,15 @@ def main(mode, input, output, name, ncores, reference_genome, mode = ['bam', 'check', 'support']\n """ - __version__ = get_distribution('bap-atac').version + __version__ = bap_version() script_dir = os.path.dirname(os.path.realpath(__file__)) click.echo(gettime() + "Starting bap2 pipeline v%s" % __version__) output = output.rstrip("/") # just for consistency - + # Determine which genomes are available - rawsg = os.popen('ls ' + script_dir + "/anno/bedtools/*.sizes").read().strip().split("\n") - supported_genomes = [x.replace(script_dir + "/anno/bedtools/chrom_", "").replace(".sizes", "") for x in rawsg] + rawsg = sorted(glob.glob(script_dir + "/anno/bedtools/*.sizes")) + supported_genomes = [x.replace(script_dir + "/anno/bedtools/chrom_", "").replace(".sizes", "") for x in rawsg] # Determine number of cores in main job if(ncores == "detect"): @@ -103,10 +96,10 @@ def main(mode, input, output, name, ncores, reference_genome, ncores = str(ncores) # Parameterize optional snakemake configuration - snakeclust = "" + snakeclust = [] njobs = int(jobs) if(njobs > 0 and cluster != ""): - snakeclust = " --jobs " + str(jobs) + " --cluster '" + cluster + "' " + snakeclust = ["--jobs", str(jobs), "--cluster", cluster] click.echo(gettime() + "Recognized flags to process jobs on a cluster.") @@ -189,13 +182,18 @@ def main(mode, input, output, name, ncores, reference_genome, click.echo(gettime() + "Splitting input bam files by chromosome for parallel processing.") click.echo(gettime() + "User specified "+ncores+" cores for parallel processing.") - line1 = 'python ' +script_dir+'/bin/python/20_names_split_filt.py --input '+p.bamfile - line2 = ' --name ' + p.name + ' --output ' + temp_filt_split + ' --barcode-tag ' - line3 = p.bead_tag + " --bedtools-reference-genome " + p.bedtoolsGenomeFile - line4 = " --mito-chr " +p.mitochr + " --ncores " + str(ncores) + " --mapq " + str(mapq) - - filt_split_cmd = line1 + line2 + line3 + line4 - os.system(filt_split_cmd) + filt_split_cmd = [ + sys.executable, script_dir + "/bin/python/20_names_split_filt.py", + "--input", p.bamfile, + "--name", p.name, + "--output", temp_filt_split, + "--barcode-tag", p.bead_tag, + "--bedtools-reference-genome", p.bedtoolsGenomeFile, + "--mito-chr", p.mitochr, + "--ncores", str(ncores), + "--mapq", str(mapq), + ] + run_cmd(filt_split_cmd) #---------------------------------------- # Step 2 - Process fragments by Snakemake @@ -203,22 +201,25 @@ def main(mode, input, output, name, ncores, reference_genome, click.echo(gettime() + "Processing bam file using a Snakemake workflow. This is the most computationally intensive step.") # Round trip the .yaml of user configuration y_s = of + "/.internal/parseltongue/bap.object.bam.yaml" + yaml_writer = YAML() + yaml_writer.default_flow_style = False with open(y_s, 'w') as yaml_file: - yaml.dump(dict(p), yaml_file, default_flow_style=False, Dumper=yaml.RoundTripDumper) - os.system("cp " + y_s + " " + logs + "/" + p.name + ".parameters.txt") - + yaml_writer.dump(dict(p), yaml_file) + shutil.copyfile(y_s, logs + "/" + p.name + ".parameters.txt") + # Assemble some log files for the snake file snake_stats = logs + "/" + p.name + ".snakemake.stats" snake_log = logs + "/" + p.name + ".snakemake.log" - - # Handle edge cases of needing to write snakemake stdout to log - if(snakemake_stdout): - snake_log_preference = "" - else: - snake_log_preference = ' &>' + snake_log - - snakecmd_chr = p.snakemake+snakeclust+' --snakefile '+script_dir+'/bin/snake/Snakefile.bap2.chr --cores '+ncores+' --config cfp="' + y_s + '" --stats '+snake_stats+snake_log_preference - os.system(snakecmd_chr) + + snakecmd_chr = [p.snakemake] + snakeclust + [ + "--snakefile", script_dir + "/bin/snake/Snakefile.bap2.chr", + "--cores", ncores, + "--config", "cfp=" + y_s, + "--stats", snake_stats, + ] + # Snakemake's own exit code is not checked here; success is verified below + # by the presence of the final .bam so the user gets an actionable message. + run_cmd(snakecmd_chr, log_file=(None if snakemake_stdout else snake_log), check=False) # Check to make sure snakemake Processing worked finalBamFile = p.output + "/final/" + p.name + ".bap.bam" @@ -236,12 +237,16 @@ def main(mode, input, output, name, ncores, reference_genome, dict_file = fin+"/"+p.name+".barcodeTranslate.tsv" - line1 = 'python ' +script_dir+'/bin/python/17_processMito.py --input '+p.bamfile - line2 = ' --output ' + mito + "/" + p.name + ".mito.bam" + " --mitochr " + p.mitochr - line3 = ' --bead-barcode ' + p.bead_tag + ' --drop-barcode ' + p.drop_tag + " --dict-file " + dict_file - mito_cmd = line1 + line2 + line3 - - os.system(mito_cmd) + mito_cmd = [ + sys.executable, script_dir + "/bin/python/17_processMito.py", + "--input", p.bamfile, + "--output", mito + "/" + p.name + ".mito.bam", + "--mitochr", p.mitochr, + "--bead-barcode", p.bead_tag, + "--drop-barcode", p.drop_tag, + "--dict-file", dict_file, + ] + run_cmd(mito_cmd) #------------------------------------------------------- diff --git a/bap/cli_bap_bulk_frag.py b/bap/cli_bap_bulk_frag.py index 2d59fe6..5bf1f21 100644 --- a/bap/cli_bap_bulk_frag.py +++ b/bap/cli_bap_bulk_frag.py @@ -3,22 +3,15 @@ import os.path import sys import shutil -import yaml -import random -import string -import itertools -import time +import glob import pysam -from pkg_resources import get_distribution -from subprocess import call, check_call from .bapHelp import * from .bapFragProjectClass import * -from ruamel import yaml -from ruamel.yaml.scalarstring import SingleQuotedScalarString as sqs +from ruamel.yaml import YAML @click.command() -@click.version_option() +@click.version_option(version=bap_version()) @click.option('--input', '-i', help='Input for bap-frag; varies by which mode is specified; for `bam`, .bam file with an index.') @click.option('--output', '-o', default="bap_out", help='Output directory for analysis; this is where everything is housed.') @@ -57,14 +50,14 @@ def main(input, output, name, ncores, reference_genome, """ - __version__ = get_distribution('bap-atac').version + __version__ = bap_version() script_dir = os.path.dirname(os.path.realpath(__file__)) click.echo(gettime() + "Starting bap-frag pipeline v%s" % __version__) - + # Determine which genomes are available - rawsg = os.popen('ls ' + script_dir + "/anno/bedtools/*.sizes").read().strip().split("\n") - supported_genomes = [x.replace(script_dir + "/anno/bedtools/chrom_", "").replace(".sizes", "") for x in rawsg] + rawsg = sorted(glob.glob(script_dir + "/anno/bedtools/*.sizes")) + supported_genomes = [x.replace(script_dir + "/anno/bedtools/chrom_", "").replace(".sizes", "") for x in rawsg] # Determine number of cores in main job if(ncores == "detect"): @@ -73,10 +66,10 @@ def main(input, output, name, ncores, reference_genome, ncores = str(ncores) # Parameterize optional snakemake configuration - snakeclust = "" + snakeclust = [] njobs = int(jobs) if(njobs > 0 and cluster != ""): - snakeclust = " --jobs " + str(jobs) + " --cluster '" + cluster + "' " + snakeclust = ["--jobs", str(jobs), "--cluster", cluster] click.echo(gettime() + "Recognized flags to process jobs on a cluster.") # Figure out if the specified reference genome is a species mix @@ -142,29 +135,40 @@ def main(input, output, name, ncores, reference_genome, click.echo(gettime() + "Splitting input bam files by chromosome for parallel processing.") click.echo(gettime() + "User specified "+ncores+" cores for parallel processing.") - line1 = 'python ' +script_dir+'/bin/python/20a_nonames_split_filt.py --input '+p.bamfile - line2 = ' --name ' + p.name + ' --output ' + temp_filt_split - line3 = " --bedtools-reference-genome " + p.bedtoolsGenomeFile - line4 = " --mito-chr " +p.mitochr + " --ncores " + str(ncores) + " --mapq " + str(mapq) - - filt_split_cmd = line1 + line2 + line3 + line4 - os.system(filt_split_cmd) - + filt_split_cmd = [ + sys.executable, script_dir + "/bin/python/20a_nonames_split_filt.py", + "--input", p.bamfile, + "--name", p.name, + "--output", temp_filt_split, + "--bedtools-reference-genome", p.bedtoolsGenomeFile, + "--mito-chr", p.mitochr, + "--ncores", str(ncores), + "--mapq", str(mapq), + ] + run_cmd(filt_split_cmd) + #---------------------------------------- # Step 2 - Process fragments by Snakemake #---------------------------------------- click.echo(gettime() + "Processing per-chromosome fragments in parallel. This is the most computationally intensive step.") # Round trip the .yaml of user configuration y_s = of + "/.internal/parseltongue/bap.object.bam.yaml" + yaml_writer = YAML() + yaml_writer.default_flow_style = False with open(y_s, 'w') as yaml_file: - yaml.dump(dict(p), yaml_file, default_flow_style=False, Dumper=yaml.RoundTripDumper) - os.system("cp " + y_s + " " + logs + "/" + p.name + ".parameters.txt") - + yaml_writer.dump(dict(p), yaml_file) + shutil.copyfile(y_s, logs + "/" + p.name + ".parameters.txt") + # Assemble some log files for the snake file snake_stats = logs + "/" + p.name + ".snakemake.stats" snake_log = logs + "/" + p.name + ".snakemake.log" - snakecmd_chr = 'snakemake'+snakeclust+' --snakefile '+script_dir+'/bin/snake/Snakefile.bap_bulk_frags --cores '+ncores+' --config cfp="' + y_s + '" --stats '+snake_stats+' &>' + snake_log - os.system(snakecmd_chr) + snakecmd_chr = ["snakemake"] + snakeclust + [ + "--snakefile", script_dir + "/bin/snake/Snakefile.bap_bulk_frags", + "--cores", ncores, + "--config", "cfp=" + y_s, + "--stats", snake_stats, + ] + run_cmd(snakecmd_chr, log_file=snake_log, check=False) #------------------------------------------------------- # Final-- remove intermediate files if necessary diff --git a/bap/cli_bap_frag.py b/bap/cli_bap_frag.py index 7f36a8c..d1f4d1c 100644 --- a/bap/cli_bap_frag.py +++ b/bap/cli_bap_frag.py @@ -3,22 +3,15 @@ import os.path import sys import shutil -import yaml -import random -import string -import itertools -import time +import glob import pysam -from pkg_resources import get_distribution -from subprocess import call, check_call from .bapHelp import * from .bapFragProjectClass import * -from ruamel import yaml -from ruamel.yaml.scalarstring import SingleQuotedScalarString as sqs +from ruamel.yaml import YAML @click.command() -@click.version_option() +@click.version_option(version=bap_version()) @click.option('--input', '-i', help='Input for bap-frag; varies by which mode is specified; for `bam`, .bam file with an index.') @click.option('--output', '-o', default="bap_out", help='Output directory for analysis; this is where everything is housed.') @@ -62,14 +55,14 @@ def main(input, output, name, ncores, reference_genome, """ - __version__ = get_distribution('bap-atac').version + __version__ = bap_version() script_dir = os.path.dirname(os.path.realpath(__file__)) click.echo(gettime() + "Starting bap-frag pipeline v%s" % __version__) - + # Determine which genomes are available - rawsg = os.popen('ls ' + script_dir + "/anno/bedtools/*.sizes").read().strip().split("\n") - supported_genomes = [x.replace(script_dir + "/anno/bedtools/chrom_", "").replace(".sizes", "") for x in rawsg] + rawsg = sorted(glob.glob(script_dir + "/anno/bedtools/*.sizes")) + supported_genomes = [x.replace(script_dir + "/anno/bedtools/chrom_", "").replace(".sizes", "") for x in rawsg] # Determine number of cores in main job if(ncores == "detect"): @@ -78,10 +71,10 @@ def main(input, output, name, ncores, reference_genome, ncores = str(ncores) # Parameterize optional snakemake configuration - snakeclust = "" + snakeclust = [] njobs = int(jobs) if(njobs > 0 and cluster != ""): - snakeclust = " --jobs " + str(jobs) + " --cluster '" + cluster + "' " + snakeclust = ["--jobs", str(jobs), "--cluster", cluster] click.echo(gettime() + "Recognized flags to process jobs on a cluster.") # Figure out if the specified reference genome is a species mix @@ -149,14 +142,18 @@ def main(input, output, name, ncores, reference_genome, click.echo(gettime() + "Splitting input bam files by chromosome for parallel processing.") click.echo(gettime() + "User specified "+ncores+" cores for parallel processing.") - line1 = 'python ' +script_dir+'/bin/python/20_names_split_filt.py --input '+p.bamfile - line2 = ' --name ' + p.name + ' --output ' + temp_filt_split + ' --barcode-tag ' - line3 = p.bead_tag + " --bedtools-reference-genome " + p.bedtoolsGenomeFile - line4 = " --mito-chr " +p.mitochr + " --ncores " + str(ncores) + " --mapq " + str(mapq) - - - filt_split_cmd = line1 + line2 + line3 + line4 - os.system(filt_split_cmd) + filt_split_cmd = [ + sys.executable, script_dir + "/bin/python/20_names_split_filt.py", + "--input", p.bamfile, + "--name", p.name, + "--output", temp_filt_split, + "--barcode-tag", p.bead_tag, + "--bedtools-reference-genome", p.bedtoolsGenomeFile, + "--mito-chr", p.mitochr, + "--ncores", str(ncores), + "--mapq", str(mapq), + ] + run_cmd(filt_split_cmd) #---------------------------------------- # Step 2 - Process fragments by Snakemake @@ -164,15 +161,22 @@ def main(input, output, name, ncores, reference_genome, click.echo(gettime() + "Processing per-chromosome fragments in parallel. This is the most computationally intensive step.") # Round trip the .yaml of user configuration y_s = of + "/.internal/parseltongue/bap.object.bam.yaml" + yaml_writer = YAML() + yaml_writer.default_flow_style = False with open(y_s, 'w') as yaml_file: - yaml.dump(dict(p), yaml_file, default_flow_style=False, Dumper=yaml.RoundTripDumper) - os.system("cp " + y_s + " " + logs + "/" + p.name + ".parameters.txt") - + yaml_writer.dump(dict(p), yaml_file) + shutil.copyfile(y_s, logs + "/" + p.name + ".parameters.txt") + # Assemble some log files for the snake file snake_stats = logs + "/" + p.name + ".snakemake.stats" snake_log = logs + "/" + p.name + ".snakemake.log" - snakecmd_chr = 'snakemake'+snakeclust+' --snakefile '+script_dir+'/bin/snake/Snakefile.bap_frags --cores '+ncores+' --config cfp="' + y_s + '" --stats '+snake_stats+' &>' + snake_log - os.system(snakecmd_chr) + snakecmd_chr = ["snakemake"] + snakeclust + [ + "--snakefile", script_dir + "/bin/snake/Snakefile.bap_frags", + "--cores", ncores, + "--config", "cfp=" + y_s, + "--stats", snake_stats, + ] + run_cmd(snakecmd_chr, log_file=snake_log, check=False) #------------------------------------------------------- diff --git a/bap/cli_reanno.py b/bap/cli_reanno.py index 54f6f52..bc96ff4 100644 --- a/bap/cli_reanno.py +++ b/bap/cli_reanno.py @@ -2,22 +2,12 @@ import os import os.path import sys -import shutil -import yaml -import random -import string -import itertools -import time import pysam -import csv -import re -from itertools import groupby from .bapHelp import * -from pkg_resources import get_distribution @click.command() -@click.version_option() +@click.version_option(version=bap_version()) @click.option('--input', '-i', help='Input bam file.') @click.option('--output', '-o', help='Output bam file.') @@ -32,7 +22,7 @@ def main(input, output, sep, tag): Caleb Lareau, clareau broadinstitute org \n """ - __version__ = get_distribution('bap-atac').version + __version__ = bap_version() script_dir = os.path.dirname(os.path.realpath(__file__)) click.echo(gettime() + "Starting re-barcoding from bap pipeline v%s" % __version__) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0e67c2a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "bap-atac" +version = "0.7.1" +description = "Bead-based scATAC-seq data Processing." +readme = "README.md" +requires-python = ">=3.9" +license = { text = "MIT" } +authors = [ + { name = "Caleb Lareau", email = "clareau@gmail.com" }, +] +keywords = ["scATAC-seq", "single-cell", "bioinformatics", "barcode", "ATAC"] +dependencies = [ + "biopython", + "fuzzysearch", + "click", + "snakemake", + "optparse-pretty", + "multiprocess", + "regex", + "pysam", + "ruamel.yaml>=0.17", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: POSIX", + "Operating System :: MacOS", + "Operating System :: Unix", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: Bio-Informatics", +] + +[project.urls] +Homepage = "https://github.com/caleblareau/bap" +Documentation = "https://github.com/caleblareau/bap/wiki" +Repository = "https://github.com/caleblareau/bap.git" + +[project.optional-dependencies] +test = ["pytest>=7"] + +[project.scripts] +bap = "bap.cli_old_dontEdit:main" +bap-barcode = "bap.barcode.cli_barcode:main" +bap-scale = "bap.barcode.cli_scaleatac:main" +bap2 = "bap.cli_bap2:main" +bap-frag = "bap.cli_bap_frag:main" +bap-bulk-frag = "bap.cli_bap_bulk_frag:main" +bap-reanno = "bap.cli_reanno:main" + +[tool.setuptools] +# Ship the non-Python assets (anno/, bin/, barcode/modes, barcode/whitelist, ...) +# declared via MANIFEST.in's `recursive-include bap *`. +include-package-data = true + +[tool.setuptools.packages.find] +include = ["bap*"] +exclude = ["tests*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "integration: end-to-end tests that require external tools (samtools, bedtools, R, snakemake); skipped automatically when they are absent", +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ef637be..0000000 --- a/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[metadata] -description-file = README.md - -[bdist_wheel] -universal=1 - diff --git a/setup.py b/setup.py deleted file mode 100644 index dac1c59..0000000 --- a/setup.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -bap: Bead-based scATAC-seq data Processing -""" -from setuptools import find_packages, setup -from distutils.core import setup, Extension - -dependencies = ['biopython','fuzzysearch','click', 'pytest', 'snakemake', 'optparse-pretty', 'multiprocess', 'regex', 'pysam', 'ruamel.yaml', 'biopython'] - -setup( - name='bap-atac', - version='0.7.1', - url='https://github.com/caleblareau/bap', - license='MIT', - author='Caleb Lareau', - author_email='clareau@gmail.com', - description='Bead-based scATAC-seq data Processing.', - long_description=__doc__, - packages=find_packages(exclude=['tests']), - include_package_data=True, - zip_safe=False, - platforms='any', - install_requires=dependencies, - entry_points={ - 'console_scripts': [ - 'bap = bap.cli_old_dontEdit:main', - 'bap-barcode = bap.barcode.cli_barcode:main', - 'bap-scale = bap.barcode.cli_scaleatac:main', - 'bap2 = bap.cli_bap2:main', - 'bap-frag = bap.cli_bap_frag:main', - 'bap-bulk-frag = bap.cli_bap_bulk_frag:main', - 'bap-reanno = bap.cli_reanno:main' - ], - }, - classifiers=[ - # As from http://pypi.python.org/pypi?%3Aaction=list_classifiers - # 'Development Status :: 1 - Planning', - # 'Development Status :: 2 - Pre-Alpha', - # 'Development Status :: 3 - Alpha', - # 'Development Status :: 4 - Beta', - 'Development Status :: 5 - Production/Stable', - # 'Development Status :: 6 - Mature', - # 'Development Status :: 7 - Inactive', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: POSIX', - 'Operating System :: MacOS', - 'Operating System :: Unix', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Topic :: Software Development :: Libraries :: Python Modules', - ] -) diff --git a/tests/README.md b/tests/README.md index 6eb4a1e..8b9f0f8 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,47 +1,100 @@ -# Testing +# Testing `bap` -Some basic use cases for testing `bap`'s execution +This directory holds the automated test suite (`pytest`) and the small data +fixtures it runs against. +## Quick start -### Basic comparison +Install the package with its test dependencies, then run the fast suite: -``` -time bap bam -i data/jaccardPairsForIGV.bam -bt XB -r hg19 -z -o bap +```bash +pip install -e .[test] -time bap2 bam -i data/jaccardPairsForIGV.bam -bt XB -r hg19 -z -o bap2 +# Fast: unit + CLI smoke tests only (no external tools needed) +pytest -m "not integration" ``` -**Note:** as discussed elsewhere, there should be no reason to use `bap`; use `bap2` for markedly better performance +The fast suite requires no bioinformatics tools and runs in under a second. It +is what CI runs on every push (see `.github/workflows/ci.yml`). -### Verify additional output for a species mix experiment or when peaks files are specified +## What each test file covers -``` -time bap2 bam -i data/small_mix.bam -bt XB -ji 0.0001 -r hg19-mm10 -z --mapq 0 -bf 10 -o SM -``` +| File | Scope | External tools | +|------|-------|----------------| +| `test_helpers.py` | Unit tests for the pure functions in `bap/bapHelp.py` (sequence math, file helpers, tool discovery, the `run_cmd` subprocess wrapper) and `mitoChr`. | none | +| `test_cli_smoke.py` | In-process `click.testing.CliRunner` checks that every active CLI imports, renders `--help`, and reports `--version`; verifies `bap2 support` lists built-in genomes. | none | +| `test_integration.py` | End-to-end `bap2 bam` run against a bundled hg19 `.bam`, asserting the final `.bap.bam`, `.barcodeTranslate.tsv`, and `.fragments.tsv.gz` are produced. | samtools, bedtools, R, snakemake | -### Test the ability to not merge when a prior is known -``` -time bap2 bam -i data/jaccardPairsForIGV.bam -bt XB -r hg19 -z -o bap2 -pf data/test.small.peaks.bed -bp data/jaccardPairsTest_sep.tsv -``` +Shared fixtures (paths to the bundled data, `small_bam`) live in `conftest.py`. + +## Running the integration tests +Integration tests are marked with `@pytest.mark.integration` and **skip +automatically** when `samtools`, `bedtools`, `R`, or `snakemake` are not on +`PATH`, so the default `pytest` run stays green on a bare machine. + +To run them, install the external tools (plus the R packages the pipeline uses: +`Rsamtools`, `GenomicAlignments`, `GenomicRanges`, `dplyr`, `data.table`) and +then: + +```bash +# Run everything, including integration tests +pytest + +# Run only the integration tests +pytest -m integration ``` -bap-barcode v2.1 -a fastq_br/biorad_v2_R1.fastq.gz -b fastq_br/biorad_v2_R2.fastq.gz -o test + +A convenient way to get the external tools is conda/mamba: + +```bash +mamba install -c bioconda -c conda-forge samtools bedtools snakemake bioconductor-genomicalignments bioconductor-rsamtools r-dplyr r-data.table ``` +## Test data + +- `data/` — small indexed `.bam` files (`jaccardPairsForIGV.bam`, + `small_mix.bam`, `test.small.bam`), a peaks bed, a whitelist, and a barcode + prior table used by the pipeline tests. +- `fastq_br/` — small BioRad and Scale-ATAC FASTQs for barcode-parsing tests. +- `for_frag/` — pre-computed annotated fragment files used to exercise the + fragment / adjacent-Tn5 code paths. + +## Manual smoke tests -## Scale-ATAC +The commands below are handy for exercising the full pipeline by hand against +the bundled data (run from this `tests/` directory). They require the external +tools listed above. + +### Basic run + +```bash +bap2 bam -i data/jaccardPairsForIGV.bam -bt XB -r hg19 -z -o bap2 ``` - bap-scale -f fastq_br/scale -s ScaleTest -o ScalePro + +> **Note:** the legacy `bap` (v1) command is deprecated — use `bap2`. + +### Species-mix / peaks-file output + +```bash +bap2 bam -i data/small_mix.bam -bt XB -ji 0.0001 -r hg19-mm10 -z --mapq 0 -bf 10 -o SM ``` -#### Docker +### Skip merging when a prior is known -Notes for CAL: +```bash +bap2 bam -i data/jaccardPairsForIGV.bam -bt XB -r hg19 -z -o bap2 \ + -pf data/test.small.peaks.bed -bp data/jaccardPairsTest_sep.tsv ``` -docker build -t caleblareau/bap bap -docker exec -it caleblareau/bap bash +### BioRad barcode parsing + +```bash +bap-barcode v2.1 -a fastq_br/biorad_v2_R1.fastq.gz -b fastq_br/biorad_v2_R2.fastq.gz -o test ``` +### Scale-ATAC barcode parsing -

+```bash +bap-scale -f fastq_br/scale -s ScaleTest -o ScalePro +``` diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6ce6cac --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +"""Shared pytest fixtures and helpers for the bap test suite.""" + +import os + +import pytest + +# Absolute path to the tests/ directory and the bundled data fixtures. +TESTS_DIR = os.path.dirname(os.path.realpath(__file__)) +DATA_DIR = os.path.join(TESTS_DIR, "data") +FASTQ_DIR = os.path.join(TESTS_DIR, "fastq_br") + + +@pytest.fixture +def data_dir(): + return DATA_DIR + + +@pytest.fixture +def small_bam(): + """A small, indexed hg19 bam bundled with the repo.""" + return os.path.join(DATA_DIR, "jaccardPairsForIGV.bam") diff --git a/tests/test_cli_smoke.py b/tests/test_cli_smoke.py new file mode 100644 index 0000000..0a9f83c --- /dev/null +++ b/tests/test_cli_smoke.py @@ -0,0 +1,45 @@ +"""Fast, hermetic smoke tests for the click CLIs. + +These invoke the command objects in-process with click's ``CliRunner`` and do +not require samtools/bedtools/R/snakemake. They verify that every active entry +point imports cleanly, renders ``--help``, and reports a version. +""" + +import pytest +from click.testing import CliRunner + +from bap.cli_bap2 import main as bap2_main +from bap.cli_bap_frag import main as bap_frag_main +from bap.cli_bap_bulk_frag import main as bap_bulk_frag_main +from bap.cli_reanno import main as bap_reanno_main +from bap.barcode.cli_barcode import main as bap_barcode_main + +ALL_COMMANDS = [ + bap2_main, + bap_frag_main, + bap_bulk_frag_main, + bap_reanno_main, + bap_barcode_main, +] + + +@pytest.mark.parametrize("command", ALL_COMMANDS) +def test_help_renders(command): + result = CliRunner().invoke(command, ["--help"]) + assert result.exit_code == 0 + assert "Usage:" in result.output + + +@pytest.mark.parametrize("command", ALL_COMMANDS) +def test_version_reports(command): + result = CliRunner().invoke(command, ["--version"]) + assert result.exit_code == 0 + assert "version" in result.output.lower() + + +def test_bap2_support_lists_builtin_genomes(): + # `support` prints the genome list and then sys.exit()s with a message, + # so the exit code is non-zero but the output holds the genome names. + result = CliRunner().invoke(bap2_main, ["support"]) + assert "hg19" in result.output + assert "GRCh38" in result.output diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..9f722de --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,184 @@ +"""Unit tests for the pure helper functions in bap.bapHelp. + +These require no external tools and run in well under a second. They also lock +in fixes for functions that were previously broken under Python 3 (e.g. +``string_hamming_distance`` used the Python-2-only ``itertools.imap``). +""" + +import os +import sys + +import pytest + +from bap import bapHelp +from bap.bapHelp import ( + bap_version, + run_cmd, + string_hamming_distance, + intersection, + findIdx, + file_len, + verify_file, + get_software_path, + rev_comp, + available_cpu_count, + gettime, +) +from bap.bap2ProjectClass import mitoChr + + +# --------------------------------------------------------------------------- +# Sequence helpers +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("seq,expected", [ + ("ACGT", "ACGT"), # reverse complement is a palindrome here + ("AAAA", "TTTT"), + ("ATCG", "CGAT"), + ("N", "N"), + ("", ""), +]) +def test_rev_comp(seq, expected): + assert rev_comp(seq) == expected + + +@pytest.mark.parametrize("a,b,expected", [ + ("karolin", "kathrin", 3), + ("AAAA", "AAAA", 0), + ("ACGT", "TGCA", 4), + ("", "", 0), +]) +def test_string_hamming_distance(a, b, expected): + assert string_hamming_distance(a, b) == expected + + +# --------------------------------------------------------------------------- +# List helpers +# --------------------------------------------------------------------------- + +def test_intersection(): + assert intersection([1, 2, 3], [2, 3, 4]) == [2, 3] + assert intersection(["chr1", "chr2"], ["chrX"]) == [] + + +def test_findIdx(): + assert findIdx(["a", "b", "c"], ["b"]) == [1] + assert findIdx(["a", "b", "c"], ["a", "c"]) == [0, 2] + + +# --------------------------------------------------------------------------- +# File helpers +# --------------------------------------------------------------------------- + +def test_file_len(tmp_path): + p = tmp_path / "lines.txt" + p.write_text("a\nb\nc\n") + assert file_len(str(p)) == 3 + + +def test_verify_file_ok(tmp_path): + p = tmp_path / "ok.txt" + p.write_text("hello\n") + assert verify_file(str(p)) == str(p) + + +def test_verify_file_missing_exits(tmp_path): + with pytest.raises(SystemExit): + verify_file(str(tmp_path / "does_not_exist.txt")) + + +# --------------------------------------------------------------------------- +# Tool discovery +# --------------------------------------------------------------------------- + +def test_get_software_path_found(): + # The Python interpreter is always discoverable via its basename on PATH + # is not guaranteed, so use an absolute-path override instead. + assert get_software_path("definitely-not-a-real-tool", sys.executable) == sys.executable + + +def test_get_software_path_missing_exits(): + with pytest.raises(SystemExit): + get_software_path("definitely-not-a-real-tool-xyz", "") + + +# --------------------------------------------------------------------------- +# Reference-genome mitochondrial chromosome naming +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("ref,expected", [ + ("hg19", "chrM"), + ("mm10", "chrM"), + ("GRCh38", "MT"), + ("hg19_mm10_c", "humanM"), + ("some_unknown_ref", "hg19_chrM"), +]) +def test_mitoChr_defaults(ref, expected): + assert mitoChr(ref, "default") == expected + + +def test_mitoChr_explicit_override(): + assert mitoChr("hg19", "myMito") == "myMito" + + +# --------------------------------------------------------------------------- +# Misc helpers +# --------------------------------------------------------------------------- + +def test_available_cpu_count(): + n = available_cpu_count() + assert isinstance(n, int) + assert n >= 1 + + +def test_bap_version_is_string(): + v = bap_version() + assert isinstance(v, str) + assert v != "" + + +def test_gettime_format(): + # Ends with the ": " separator used throughout the log messages + assert gettime().endswith(": ") + + +# --------------------------------------------------------------------------- +# run_cmd: the subprocess wrapper that replaced os.system() +# --------------------------------------------------------------------------- + +def test_run_cmd_success(): + result = run_cmd([sys.executable, "-c", "pass"]) + assert result.returncode == 0 + + +def test_run_cmd_failure_exits(): + with pytest.raises(SystemExit): + run_cmd([sys.executable, "-c", "import sys; sys.exit(3)"]) + + +def test_run_cmd_failure_no_check(): + result = run_cmd([sys.executable, "-c", "import sys; sys.exit(3)"], check=False) + assert result.returncode == 3 + + +def test_run_cmd_captures_combined_log(tmp_path): + log = tmp_path / "out.log" + run_cmd( + [sys.executable, "-c", "import sys; print('to_out'); print('to_err', file=sys.stderr)"], + log_file=str(log), + ) + contents = log.read_text() + assert "to_out" in contents + assert "to_err" in contents + + +def test_run_cmd_stderr_only_log(tmp_path): + log = tmp_path / "err.log" + run_cmd( + [sys.executable, "-c", "import sys; print('to_out'); print('to_err', file=sys.stderr)"], + log_file=str(log), + log_stderr_only=True, + ) + contents = log.read_text() + assert "to_err" in contents + assert "to_out" not in contents diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..a2341cf --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,42 @@ +"""End-to-end integration tests for the bap2 pipeline. + +These exercise the real workflow and therefore require the external tools the +pipeline orchestrates (samtools, bedtools, R + pipeline R packages, snakemake). +They are automatically skipped when those tools are not on PATH, so the default +``pytest`` run stays green on a bare machine. Run them explicitly with: + + pytest -m integration +""" + +import shutil + +import pytest +from click.testing import CliRunner + +from bap.cli_bap2 import main as bap2_main + +# External tools the full pipeline shells out to; skip when any are missing. +REQUIRED_TOOLS = ("samtools", "bedtools", "R", "snakemake") +_missing = [t for t in REQUIRED_TOOLS if shutil.which(t) is None] +requires_external_tools = pytest.mark.skipif( + bool(_missing), reason="external tools not on PATH: " + ", ".join(_missing) +) + + +@pytest.mark.integration +@requires_external_tools +def test_bap2_bam_end_to_end(small_bam, tmp_path): + outdir = tmp_path / "bap2_out" + result = CliRunner().invoke( + bap2_main, + ["bam", "-i", small_bam, "-bt", "XB", "-r", "hg19", "-z", "-o", str(outdir)], + ) + + # Surface pipeline output if something went wrong, to aid debugging. + assert result.exit_code == 0, result.output + + name = "jaccardPairsForIGV" # derived from the input .bam prefix + final_dir = outdir / "final" + assert (final_dir / (name + ".bap.bam")).exists() + assert (final_dir / (name + ".barcodeTranslate.tsv")).exists() + assert (final_dir / (name + ".fragments.tsv.gz")).exists()