From 235cbe8258ac3fd585f14ec3941c75be5d96e24d Mon Sep 17 00:00:00 2001 From: kvnjng Date: Wed, 6 Nov 2019 14:57:34 -0500 Subject: [PATCH 1/5] change hash-bang lines to python 3 env and run any subprocesses in Python 3 --- LDlink/LDassoc.py | 12 ++++++------ LDlink/LDassoc_plot_sub.py | 8 ++++---- LDlink/LDhap.py | 2 +- LDlink/LDmatrix.py | 4 ++-- LDlink/LDpair.py | 2 +- LDlink/LDpop.py | 2 +- LDlink/LDproxy.py | 12 ++++++------ LDlink/LDproxy_plot_sub.py | 8 ++++---- LDlink/RegisterAPI.py | 2 +- LDlink/SNPchip.py | 2 +- LDlink/SNPclip.py | 2 +- LDlink/UnlockStaleTokens.py | 2 +- 12 files changed, 29 insertions(+), 29 deletions(-) diff --git a/LDlink/LDassoc.py b/LDlink/LDassoc.py index 30e7dd20..57319727 100755 --- a/LDlink/LDassoc.py +++ b/LDlink/LDassoc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import yaml import csv import json @@ -552,13 +552,13 @@ def get_coords_gene(gene_raw): print("Create LDassoc_sub subprocesses") for i in range(threads): if i==min(range(threads)) and i==max(range(threads)): - command="python LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords)+" "+request+" "+str(i) + command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords)+" "+request+" "+str(i) elif i==min(range(threads)): - command="python LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[:block])+" "+request+" "+str(i) + command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[:block])+" "+request+" "+str(i) elif i==max(range(threads)): - command="python LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:])+" "+request+" "+str(i) + command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:])+" "+request+" "+str(i) else: - command="python LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:block*(i+1)])+" "+request+" "+str(i) + command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:block*(i+1)])+" "+request+" "+str(i) commands.append(command) @@ -1122,7 +1122,7 @@ def get_output(process): if web: # Open thread for high quality image exports print("Open thread for high quality image exports.") - command = "python LDassoc_plot_sub.py " + tmp_dir + 'assoc_args' + request + ".json" + " " + file + " " + region + " " + pop + " " + request + " " + myargsName + " " + myargsOrigin + command = "python3 LDassoc_plot_sub.py " + tmp_dir + 'assoc_args' + request + ".json" + " " + file + " " + region + " " + pop + " " + request + " " + myargsName + " " + myargsOrigin subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) diff --git a/LDlink/LDassoc_plot_sub.py b/LDlink/LDassoc_plot_sub.py index bdae4834..ff77826f 100644 --- a/LDlink/LDassoc_plot_sub.py +++ b/LDlink/LDassoc_plot_sub.py @@ -390,13 +390,13 @@ def get_coords_gene(gene_raw): commands=[] for i in range(threads): if i==min(range(threads)) and i==max(range(threads)): - command="python LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords)+" "+request+" "+str(i) + command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords)+" "+request+" "+str(i) elif i==min(range(threads)): - command="python LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[:block])+" "+request+" "+str(i) + command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[:block])+" "+request+" "+str(i) elif i==max(range(threads)): - command="python LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:])+" "+request+" "+str(i) + command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:])+" "+request+" "+str(i) else: - command="python LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:block*(i+1)])+" "+request+" "+str(i) + command="python3 LDassoc_sub.py "+snp+" "+chromosome+" "+"_".join(assoc_coords[(block*i)+1:block*(i+1)])+" "+request+" "+str(i) commands.append(command) diff --git a/LDlink/LDhap.py b/LDlink/LDhap.py index a278d389..7e3b274b 100755 --- a/LDlink/LDhap.py +++ b/LDlink/LDhap.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import yaml import json import math diff --git a/LDlink/LDmatrix.py b/LDlink/LDmatrix.py index ead9ae30..14a55ca2 100755 --- a/LDlink/LDmatrix.py +++ b/LDlink/LDmatrix.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import yaml import json import math @@ -907,7 +907,7 @@ def set_alleles(a1, a2): # Generate high quality images only if accessed via web instance if web: # Open thread for high quality image exports - command = "python LDmatrix_plot_sub.py " + \ + command = "python3 LDmatrix_plot_sub.py " + \ snplst + " " + pop + " " + request + " " + r2_d subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) diff --git a/LDlink/LDpair.py b/LDlink/LDpair.py index 3cd0326c..03f0de51 100755 --- a/LDlink/LDpair.py +++ b/LDlink/LDpair.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import yaml import json import math diff --git a/LDlink/LDpop.py b/LDlink/LDpop.py index 97586828..1703a209 100755 --- a/LDlink/LDpop.py +++ b/LDlink/LDpop.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import yaml import json import math diff --git a/LDlink/LDproxy.py b/LDlink/LDproxy.py index 5f8c5cc4..9e71565c 100755 --- a/LDlink/LDproxy.py +++ b/LDlink/LDproxy.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import yaml import csv import json @@ -249,18 +249,18 @@ def replace_coord_rsid(snp): commands = [] for i in range(threads): if i == min(range(threads)) and i == max(range(threads)): - command = "python LDproxy_sub.py " + str(web) + " " + snp + " " + \ + command = "python3 LDproxy_sub.py " + str(web) + " " + snp + " " + \ snp_coord['chromosome'] + " " + str(coord1) + " " + \ str(coord2) + " " + request + " " + str(i) elif i == min(range(threads)): - command = "python LDproxy_sub.py " + str(web) + " " + snp + " " + \ + command = "python3 LDproxy_sub.py " + str(web) + " " + snp + " " + \ snp_coord['chromosome'] + " " + str(coord1) + " " + \ str(coord1 + block) + " " + request + " " + str(i) elif i == max(range(threads)): - command = "python LDproxy_sub.py " + str(web) + " " + snp + " " + snp_coord['chromosome'] + " " + str( + command = "python3 LDproxy_sub.py " + str(web) + " " + snp + " " + snp_coord['chromosome'] + " " + str( coord1 + (block * i) + 1) + " " + str(coord2) + " " + request + " " + str(i) else: - command = "python LDproxy_sub.py " + str(web) + " " + snp + " " + snp_coord['chromosome'] + " " + str(coord1 + ( + command = "python3 LDproxy_sub.py " + str(web) + " " + snp + " " + snp_coord['chromosome'] + " " + str(coord1 + ( block * i) + 1) + " " + str(coord1 + (block * (i + 1))) + " " + request + " " + str(i) commands.append(command) @@ -818,7 +818,7 @@ def get_output(process): # Generate high quality images only if accessed via web instance if web: # Open thread for high quality image exports - command = "python LDproxy_plot_sub.py " + snp + " " + pop + " " + request + " " + r2_d + command = "python3 LDproxy_plot_sub.py " + snp + " " + pop + " " + request + " " + r2_d subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) ########################### diff --git a/LDlink/LDproxy_plot_sub.py b/LDlink/LDproxy_plot_sub.py index c8b521aa..bf7264ae 100644 --- a/LDlink/LDproxy_plot_sub.py +++ b/LDlink/LDproxy_plot_sub.py @@ -179,18 +179,18 @@ def replace_coord_rsid(db, snp): commands = [] for i in range(threads): if i == min(range(threads)) and i == max(range(threads)): - command = "python LDproxy_sub.py " + "True " + snp + " " + \ + command = "python3 LDproxy_sub.py " + "True " + snp + " " + \ snp_coord['chromosome'] + " " + str(coord1) + " " + \ str(coord2) + " " + request + " " + str(i) elif i == min(range(threads)): - command = "python LDproxy_sub.py " + "True " + snp + " " + \ + command = "python3 LDproxy_sub.py " + "True " + snp + " " + \ snp_coord['chromosome'] + " " + str(coord1) + " " + \ str(coord1 + block) + " " + request + " " + str(i) elif i == max(range(threads)): - command = "python LDproxy_sub.py " + "True " + snp + " " + snp_coord['chromosome'] + " " + str( + command = "python3 LDproxy_sub.py " + "True " + snp + " " + snp_coord['chromosome'] + " " + str( coord1 + (block * i) + 1) + " " + str(coord2) + " " + request + " " + str(i) else: - command = "python LDproxy_sub.py " + "True " + snp + " " + snp_coord['chromosome'] + " " + str(coord1 + ( + command = "python3 LDproxy_sub.py " + "True " + snp + " " + snp_coord['chromosome'] + " " + str(coord1 + ( block * i) + 1) + " " + str(coord1 + (block * (i + 1))) + " " + request + " " + str(i) commands.append(command) diff --git a/LDlink/RegisterAPI.py b/LDlink/RegisterAPI.py index 12b3d764..66f06e6a 100644 --- a/LDlink/RegisterAPI.py +++ b/LDlink/RegisterAPI.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import json import os.path import binascii diff --git a/LDlink/SNPchip.py b/LDlink/SNPchip.py index dbb1a449..6e7c01bc 100755 --- a/LDlink/SNPchip.py +++ b/LDlink/SNPchip.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 ########### # SNPchip # diff --git a/LDlink/SNPclip.py b/LDlink/SNPclip.py index e901a59a..61e58f55 100755 --- a/LDlink/SNPclip.py +++ b/LDlink/SNPclip.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import yaml import json import math diff --git a/LDlink/UnlockStaleTokens.py b/LDlink/UnlockStaleTokens.py index a0e5bedc..7f4b4f3c 100644 --- a/LDlink/UnlockStaleTokens.py +++ b/LDlink/UnlockStaleTokens.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import datetime import dateutil.parser import yaml From 02605b5b1f1b8dbd0eb42a449a5eaf09502e0d1e Mon Sep 17 00:00:00 2001 From: kvnjng Date: Wed, 6 Nov 2019 15:11:28 -0500 Subject: [PATCH 2/5] run LDassoc subprocess plot in Python 3 --- LDlink/LDassoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LDlink/LDassoc.py b/LDlink/LDassoc.py index 57319727..b861cf9d 100755 --- a/LDlink/LDassoc.py +++ b/LDlink/LDassoc.py @@ -1264,7 +1264,7 @@ def get_output(process): if web: # Open thread for high quality image exports print("Open thread for high quality image exports.") - command = "python LDassoc_plot_sub.py " + tmp_dir + 'assoc_args' + request + ".json" + " " + file + " " + region + " " + pop + " " + request + " " + myargsName + " " + myargsOrigin + command = "python3 LDassoc_plot_sub.py " + tmp_dir + 'assoc_args' + request + ".json" + " " + file + " " + region + " " + pop + " " + request + " " + myargsName + " " + myargsOrigin subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) ########################### From cb7b7e3b9951806ee9a60554fdb7ab98a40af775 Mon Sep 17 00:00:00 2001 From: kvnjng Date: Wed, 6 Nov 2019 15:34:15 -0500 Subject: [PATCH 3/5] fix block floored integer as index --- LDlink/LDassoc_plot_sub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LDlink/LDassoc_plot_sub.py b/LDlink/LDassoc_plot_sub.py index ff77826f..32029112 100644 --- a/LDlink/LDassoc_plot_sub.py +++ b/LDlink/LDassoc_plot_sub.py @@ -386,7 +386,7 @@ def get_coords_gene(gene_raw): else: threads=4 - block=len(assoc_coords)/threads + block=len(assoc_coords)//threads commands=[] for i in range(threads): if i==min(range(threads)) and i==max(range(threads)): From 2552a6cac556af4805616462b955093ab1aa1c06 Mon Sep 17 00:00:00 2001 From: kvnjng Date: Wed, 6 Nov 2019 17:15:13 -0500 Subject: [PATCH 4/5] debug catch and print any errors from subprocesses --- LDlink/LDproxy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/LDlink/LDproxy.py b/LDlink/LDproxy.py index 9e71565c..0b1a6464 100755 --- a/LDlink/LDproxy.py +++ b/LDlink/LDproxy.py @@ -265,11 +265,13 @@ def replace_coord_rsid(snp): commands.append(command) processes = [subprocess.Popen( - command, shell=True, stdout=subprocess.PIPE) for command in commands] + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for command in commands] # collect output in parallel def get_output(process): - return process.communicate()[0].splitlines() + stdout, stderr = process.communicate() + print("stderr", stderr) + return stdout[0].splitlines() if not hasattr(threading.current_thread(), "_children"): threading.current_thread()._children = weakref.WeakKeyDictionary() From 75ca92b8cfb8efe351732a8764231f76e37eeeef Mon Sep 17 00:00:00 2001 From: kvnjng Date: Wed, 6 Nov 2019 17:53:58 -0500 Subject: [PATCH 5/5] debug remove stderr pipe --- LDlink/LDproxy.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/LDlink/LDproxy.py b/LDlink/LDproxy.py index 0b1a6464..9e71565c 100755 --- a/LDlink/LDproxy.py +++ b/LDlink/LDproxy.py @@ -265,13 +265,11 @@ def replace_coord_rsid(snp): commands.append(command) processes = [subprocess.Popen( - command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for command in commands] + command, shell=True, stdout=subprocess.PIPE) for command in commands] # collect output in parallel def get_output(process): - stdout, stderr = process.communicate() - print("stderr", stderr) - return stdout[0].splitlines() + return process.communicate()[0].splitlines() if not hasattr(threading.current_thread(), "_children"): threading.current_thread()._children = weakref.WeakKeyDictionary()