Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion mocks/DDrppi_mocks/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a
LIBSRC := countpairs_rp_pi_mocks.c countpairs_rp_pi_mocks_impl_double.c countpairs_rp_pi_mocks_impl_float.c \
$(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \
$(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \
$(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c
LIBRARY_HEADERS := $(LIBNAME).h

Expand Down
2 changes: 1 addition & 1 deletion mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ static inline int countpairs_rp_pi_mocks_avx512_intrinsics_DOUBLE(const int64_t
const AVX512_FLOATS m_sqr_rpmin = AVX512_SET_FLOAT(sqr_rpmin);

for(int64_t j=n_off;j<N1;j+=AVX512_NVEC){
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j];
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U);
const AVX512_FLOATS m_x2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1);
const AVX512_FLOATS m_y2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1);
const AVX512_FLOATS m_z2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1);
Expand Down
2 changes: 1 addition & 1 deletion mocks/DDsmu_mocks/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a
LIBSRC := countpairs_s_mu_mocks.c countpairs_s_mu_mocks_impl_double.c countpairs_s_mu_mocks_impl_float.c \
$(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \
$(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \
$(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c
LIBRARY_HEADERS := $(LIBNAME).h

Expand Down
2 changes: 1 addition & 1 deletion mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ static inline int countpairs_s_mu_mocks_avx512_intrinsics_DOUBLE(const int64_t N
const AVX512_FLOATS m_sqr_smin = AVX512_SET_FLOAT(sqr_smin);
const AVX512_FLOATS m_sqr_mumax = AVX512_SET_FLOAT(sqr_mumax);
for(int64_t j=n_off;j<N1;j+=AVX512_NVEC){
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j];
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U);
const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1);
const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1);
const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1);
Expand Down
2 changes: 1 addition & 1 deletion mocks/DDtheta_mocks/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a
LIBSRC := countpairs_theta_mocks.c countpairs_theta_mocks_impl_float.c countpairs_theta_mocks_impl_double.c \
$(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \
$(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c
LIBRARY_HEADERS := $(LIBNAME).h

TARGET := DDtheta_mocks
Expand Down
4 changes: 2 additions & 2 deletions mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src
Original file line number Diff line number Diff line change
Expand Up @@ -1027,7 +1027,7 @@ static inline int countpairs_theta_mocks_avx512_intrinsics_DOUBLE(const int64_t

const AVX512_FLOATS m_max_dz = AVX512_SET_FLOAT(max_dz);
for(int64_t j=n_off;j<N1;j+=AVX512_NVEC) {
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j];
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U);
const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1);
const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1);
const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1);
Expand Down Expand Up @@ -1099,7 +1099,7 @@ static inline int countpairs_theta_mocks_avx512_intrinsics_DOUBLE(const int64_t
// at the beginning of the loop.
for(int kbin=nthetabin-1;kbin>=1;kbin--) {
const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_costheta,m_costheta_upp[kbin-1],_CMP_LE_OQ);
npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask];
npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask);
if(need_thetaavg || need_weightavg) {
m_thetabin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_thetabin, AVX512_SET_INT(kbin));
}
Expand Down
2 changes: 1 addition & 1 deletion mocks/vpf_mocks/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ LIBRARY:=lib$(LIBNAME).a
LIBSRC:=countspheres_mocks.c countspheres_mocks_impl_float.c countspheres_mocks_impl_double.c \
$(UTILS_DIR)/gridlink_impl_float.c $(UTILS_DIR)/gridlink_impl_double.c \
$(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \
$(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c
LIBRARY_HEADERS := $(LIBNAME).h

Expand Down
6 changes: 3 additions & 3 deletions mocks/vpf_mocks/vpf_mocks_kernels.c.src
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ static inline int vpf_mocks_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE *X
DOUBLE *localz2 = (DOUBLE *) Z;

for(int64_t j=0;j<np;j+=AVX512_NVEC) {
AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[np-j];
AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[np-j];

const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2);
const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2);
Expand All @@ -79,12 +79,12 @@ static inline int vpf_mocks_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE *X
const AVX512_MASK m_mask1 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k],_CMP_LT_OS);
const AVX512_MASK m_mask2 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k-1],_CMP_GE_OS);
const AVX512_MASK m_bin_mask = AVX512_MASK_BITWISE_AND(m_mask1,m_mask2);
counts[k] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask];
counts[k] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask);
m_mask_left = AVX512_MASK_BITWISE_AND_NOT(m_bin_mask, m_mask_left);//ANDNOT(X, Y) -> NOT X AND Y
if(m_mask_left == 0) {
break;
} else if(k==1){
counts[0] += bits_set_in_avx512_mask_DOUBLE[m_mask_left];
counts[0] += AVX512_BIT_COUNT_UNSIGNED_INT(m_mask_left);
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion theory/DD/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a
LIBSRC := countpairs.c countpairs_impl_double.c countpairs_impl_float.c \
$(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \
$(UTILS_DIR)/gridlink_utils_double.c $(UTILS_DIR)/gridlink_utils_float.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c
LIBRARY_HEADERS := $(LIBNAME).h

TARGET := DD
Expand Down
6 changes: 3 additions & 3 deletions theory/DD/countpairs_kernels.c.src
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ static inline int countpairs_avx512_intrinsics_DOUBLE(const int64_t N0, DOUBLE *
const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos);

for(int64_t j=n_off;j<N1;j+=AVX512_NVEC) {
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j];
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U);
const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1);
const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1);
const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1);
Expand Down Expand Up @@ -222,7 +222,7 @@ static inline int countpairs_avx512_intrinsics_DOUBLE(const int64_t N0, DOUBLE *
// at the beginning of the loop.
for(int kbin=nbin-1;kbin>=1;kbin--) {
const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, r2,m_rupp_sqr[kbin-1],_CMP_GE_OS);
npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask];
npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask);
if(need_rpavg || need_weightavg) {
m_rpbin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_rpbin, AVX512_SET_INT(kbin));
}
Expand Down Expand Up @@ -972,7 +972,7 @@ static inline int countpairs_fallback_DOUBLE(const int64_t N0, DOUBLE *x0, DOUBL
for(int w = 0; w < pair.num_weights; w++){
pair.weights1[w].d = *(weights1->weights[w] + n_off + j);
}

pair.dx.d = dx;
pair.dy.d = dy;
pair.dz.d = dz;
Expand Down
2 changes: 1 addition & 1 deletion theory/DDrppi/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ LIBRARY := libcountpairs_rp_pi.a
LIBSRC := countpairs_rp_pi.c countpairs_rp_pi_impl_double.c countpairs_rp_pi_impl_float.c \
$(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \
$(UTILS_DIR)/gridlink_utils_double.c $(UTILS_DIR)/gridlink_utils_float.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c
LIBRARY_HEADERS := countpairs_rp_pi.h

TARGETSRC := DDrppi.c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC)
Expand Down
2 changes: 1 addition & 1 deletion theory/DDrppi/countpairs_rp_pi_kernels.c.src
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ static inline int countpairs_rp_pi_avx512_intrinsics_DOUBLE(const int64_t N0, DO
}

for(int64_t j=n_off;j<N1;j+=AVX512_NVEC){
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j];
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U);
const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1);
const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1);
const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1);
Expand Down
2 changes: 1 addition & 1 deletion theory/DDsmu/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ LIBRARY := libcountpairs_s_mu.a
LIBSRC := countpairs_s_mu.c countpairs_s_mu_impl_double.c countpairs_s_mu_impl_float.c \
$(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \
$(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c
LIBRARY_HEADERS := countpairs_s_mu.h

TARGETSRC := DDsmu.c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC)
Expand Down
2 changes: 1 addition & 1 deletion theory/DDsmu/countpairs_s_mu_kernels.c.src
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ static inline int countpairs_s_mu_avx512_intrinsics_DOUBLE(const int64_t N0, DOU
}

for(int64_t j=n_off;j<N1;j+=AVX512_NVEC){
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j];
AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U);
const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1);
const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1);
const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1);
Expand Down
2 changes: 1 addition & 1 deletion theory/vpf/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ LIBSRC := countspheres.c countspheres_impl_double.c countspheres_impl_float.c \
$(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \
$(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c \
$(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c
$(UTILS_DIR)/cpu_features.c

TARGET := vpf
TARGETSRC := $(TARGET).c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC)
Expand Down
6 changes: 3 additions & 3 deletions theory/vpf/vpf_kernels.c.src
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ static inline int vpf_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE * restr
DOUBLE *localz2 = (DOUBLE *) Z;

for(int64_t j=0;j<np;j+=AVX512_NVEC) {
AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[np-j];
AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[np-j];

const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2);
const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2);
Expand All @@ -75,12 +75,12 @@ static inline int vpf_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE * restr
const AVX512_MASK m_mask1 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k],_CMP_LT_OS);
const AVX512_MASK m_mask2 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k-1],_CMP_GE_OS);
const AVX512_MASK m_bin_mask = AVX512_MASK_BITWISE_AND(m_mask1,m_mask2);
counts[k] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask];
counts[k] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask);
m_mask_left = AVX512_MASK_BITWISE_AND_NOT(m_bin_mask, m_mask_left);//ANDNOT(X, Y) -> NOT X AND Y
if(m_mask_left == 0) {
break;
} else if(k==1){
counts[0] += bits_set_in_avx512_mask_DOUBLE[m_mask_left];
counts[0] += AVX512_BIT_COUNT_UNSIGNED_INT(m_mask_left);
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion theory/wp/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ LIBRARY := lib$(LIBNAME).a
LIBSRC := countpairs_wp.c countpairs_wp_impl_double.c countpairs_wp_impl_float.c \
$(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \
$(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c
$(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c
LIBRARY_HEADERS := $(LIBNAME).h

TARGET := wp
Expand Down
Loading
Loading