diff --git a/mocks/DDrppi_mocks/Makefile b/mocks/DDrppi_mocks/Makefile index 8c36b65f..f6e286dc 100644 --- a/mocks/DDrppi_mocks/Makefile +++ b/mocks/DDrppi_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_rp_pi_mocks.c countpairs_rp_pi_mocks_impl_double.c countpairs_rp_pi_mocks_impl_float.c \ $(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \ + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \ $(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c LIBRARY_HEADERS := $(LIBNAME).h diff --git a/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src b/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src index 407b2ef9..a61ab06c 100644 --- a/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src +++ b/mocks/DDrppi_mocks/countpairs_rp_pi_mocks_kernels.c.src @@ -178,7 +178,7 @@ static inline int countpairs_rp_pi_mocks_avx512_intrinsics_DOUBLE(const int64_t const AVX512_FLOATS m_sqr_rpmin = AVX512_SET_FLOAT(sqr_rpmin); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/mocks/DDsmu_mocks/Makefile b/mocks/DDsmu_mocks/Makefile index 3642b932..f5ca84f1 100644 --- a/mocks/DDsmu_mocks/Makefile +++ b/mocks/DDsmu_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_s_mu_mocks.c countpairs_s_mu_mocks_impl_double.c countpairs_s_mu_mocks_impl_float.c \ $(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \ + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \ $(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c LIBRARY_HEADERS := $(LIBNAME).h diff --git a/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src b/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src index 643fe83b..c6b0e4bf 100644 --- a/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src +++ b/mocks/DDsmu_mocks/countpairs_s_mu_mocks_kernels.c.src @@ -172,7 +172,7 @@ static inline int countpairs_s_mu_mocks_avx512_intrinsics_DOUBLE(const int64_t N const AVX512_FLOATS m_sqr_smin = AVX512_SET_FLOAT(sqr_smin); const AVX512_FLOATS m_sqr_mumax = AVX512_SET_FLOAT(sqr_mumax); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/mocks/DDtheta_mocks/Makefile b/mocks/DDtheta_mocks/Makefile index 0ef90bbf..61a31924 100644 --- a/mocks/DDtheta_mocks/Makefile +++ b/mocks/DDtheta_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_theta_mocks.c countpairs_theta_mocks_impl_float.c countpairs_theta_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_mocks_impl_float.c $(UTILS_DIR)/gridlink_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := $(LIBNAME).h TARGET := DDtheta_mocks diff --git a/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src b/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src index acac3954..13691d4e 100644 --- a/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src +++ b/mocks/DDtheta_mocks/countpairs_theta_mocks_kernels.c.src @@ -1027,7 +1027,7 @@ static inline int countpairs_theta_mocks_avx512_intrinsics_DOUBLE(const int64_t const AVX512_FLOATS m_max_dz = AVX512_SET_FLOAT(max_dz); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); @@ -1099,7 +1099,7 @@ static inline int countpairs_theta_mocks_avx512_intrinsics_DOUBLE(const int64_t // at the beginning of the loop. for(int kbin=nthetabin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_costheta,m_costheta_upp[kbin-1],_CMP_LE_OQ); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_thetaavg || need_weightavg) { m_thetabin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_thetabin, AVX512_SET_INT(kbin)); } diff --git a/mocks/vpf_mocks/Makefile b/mocks/vpf_mocks/Makefile index 808cc938..40df0a99 100644 --- a/mocks/vpf_mocks/Makefile +++ b/mocks/vpf_mocks/Makefile @@ -13,7 +13,7 @@ LIBRARY:=lib$(LIBNAME).a LIBSRC:=countspheres_mocks.c countspheres_mocks_impl_float.c countspheres_mocks_impl_double.c \ $(UTILS_DIR)/gridlink_impl_float.c $(UTILS_DIR)/gridlink_impl_double.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c \ + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c \ $(UTILS_DIR)/set_cosmo_dist.c $(UTILS_DIR)/cosmology_params.c LIBRARY_HEADERS := $(LIBNAME).h diff --git a/mocks/vpf_mocks/vpf_mocks_kernels.c.src b/mocks/vpf_mocks/vpf_mocks_kernels.c.src index b559ef76..79555011 100644 --- a/mocks/vpf_mocks/vpf_mocks_kernels.c.src +++ b/mocks/vpf_mocks/vpf_mocks_kernels.c.src @@ -53,7 +53,7 @@ static inline int vpf_mocks_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE *X DOUBLE *localz2 = (DOUBLE *) Z; for(int64_t j=0;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[np-j]; + AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[np-j]; const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); @@ -79,12 +79,12 @@ static inline int vpf_mocks_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE *X const AVX512_MASK m_mask1 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k],_CMP_LT_OS); const AVX512_MASK m_mask2 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k-1],_CMP_GE_OS); const AVX512_MASK m_bin_mask = AVX512_MASK_BITWISE_AND(m_mask1,m_mask2); - counts[k] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + counts[k] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); m_mask_left = AVX512_MASK_BITWISE_AND_NOT(m_bin_mask, m_mask_left);//ANDNOT(X, Y) -> NOT X AND Y if(m_mask_left == 0) { break; } else if(k==1){ - counts[0] += bits_set_in_avx512_mask_DOUBLE[m_mask_left]; + counts[0] += AVX512_BIT_COUNT_UNSIGNED_INT(m_mask_left); } } } diff --git a/theory/DD/Makefile b/theory/DD/Makefile index 330b9770..d4e0985b 100644 --- a/theory/DD/Makefile +++ b/theory/DD/Makefile @@ -13,7 +13,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs.c countpairs_impl_double.c countpairs_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_double.c $(UTILS_DIR)/gridlink_utils_float.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := $(LIBNAME).h TARGET := DD diff --git a/theory/DD/countpairs_kernels.c.src b/theory/DD/countpairs_kernels.c.src index c841fab8..6acc27c7 100644 --- a/theory/DD/countpairs_kernels.c.src +++ b/theory/DD/countpairs_kernels.c.src @@ -147,7 +147,7 @@ static inline int countpairs_avx512_intrinsics_DOUBLE(const int64_t N0, DOUBLE * const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos); for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); @@ -222,7 +222,7 @@ static inline int countpairs_avx512_intrinsics_DOUBLE(const int64_t N0, DOUBLE * // at the beginning of the loop. for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, r2,m_rupp_sqr[kbin-1],_CMP_GE_OS); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_rpavg || need_weightavg) { m_rpbin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_rpbin, AVX512_SET_INT(kbin)); } @@ -972,7 +972,7 @@ static inline int countpairs_fallback_DOUBLE(const int64_t N0, DOUBLE *x0, DOUBL for(int w = 0; w < pair.num_weights; w++){ pair.weights1[w].d = *(weights1->weights[w] + n_off + j); } - + pair.dx.d = dx; pair.dy.d = dy; pair.dz.d = dz; diff --git a/theory/DDrppi/Makefile b/theory/DDrppi/Makefile index 0f6f5c16..7555a8db 100644 --- a/theory/DDrppi/Makefile +++ b/theory/DDrppi/Makefile @@ -13,7 +13,7 @@ LIBRARY := libcountpairs_rp_pi.a LIBSRC := countpairs_rp_pi.c countpairs_rp_pi_impl_double.c countpairs_rp_pi_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_double.c $(UTILS_DIR)/gridlink_utils_float.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := countpairs_rp_pi.h TARGETSRC := DDrppi.c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) diff --git a/theory/DDrppi/countpairs_rp_pi_kernels.c.src b/theory/DDrppi/countpairs_rp_pi_kernels.c.src index b0af4624..45a5a62f 100644 --- a/theory/DDrppi/countpairs_rp_pi_kernels.c.src +++ b/theory/DDrppi/countpairs_rp_pi_kernels.c.src @@ -162,7 +162,7 @@ static inline int countpairs_rp_pi_avx512_intrinsics_DOUBLE(const int64_t N0, DO } for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/theory/DDsmu/Makefile b/theory/DDsmu/Makefile index d38d0947..76168c66 100644 --- a/theory/DDsmu/Makefile +++ b/theory/DDsmu/Makefile @@ -13,7 +13,7 @@ LIBRARY := libcountpairs_s_mu.a LIBSRC := countpairs_s_mu.c countpairs_s_mu_impl_double.c countpairs_s_mu_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := countpairs_s_mu.h TARGETSRC := DDsmu.c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) diff --git a/theory/DDsmu/countpairs_s_mu_kernels.c.src b/theory/DDsmu/countpairs_s_mu_kernels.c.src index cd590cca..9e567b24 100644 --- a/theory/DDsmu/countpairs_s_mu_kernels.c.src +++ b/theory/DDsmu/countpairs_s_mu_kernels.c.src @@ -178,7 +178,7 @@ static inline int countpairs_s_mu_avx512_intrinsics_DOUBLE(const int64_t N0, DOU } for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); diff --git a/theory/vpf/Makefile b/theory/vpf/Makefile index ad3de9fc..c02c3dab 100644 --- a/theory/vpf/Makefile +++ b/theory/vpf/Makefile @@ -14,7 +14,7 @@ LIBSRC := countspheres.c countspheres_impl_double.c countspheres_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c \ - $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/cpu_features.c TARGET := vpf TARGETSRC := $(TARGET).c $(IO_DIR)/ftread.c $(IO_DIR)/io.c $(LIBSRC) diff --git a/theory/vpf/vpf_kernels.c.src b/theory/vpf/vpf_kernels.c.src index 6e0f74c3..c173e99f 100644 --- a/theory/vpf/vpf_kernels.c.src +++ b/theory/vpf/vpf_kernels.c.src @@ -48,7 +48,7 @@ static inline int vpf_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE * restr DOUBLE *localz2 = (DOUBLE *) Z; for(int64_t j=0;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[np-j]; + AVX512_MASK m_mask_left = (np - j) >= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[np-j]; const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); @@ -75,12 +75,12 @@ static inline int vpf_avx512_intrinsics_DOUBLE(const int64_t np, DOUBLE * restr const AVX512_MASK m_mask1 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k],_CMP_LT_OS); const AVX512_MASK m_mask2 = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_r2,m_rupp_sqr[k-1],_CMP_GE_OS); const AVX512_MASK m_bin_mask = AVX512_MASK_BITWISE_AND(m_mask1,m_mask2); - counts[k] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + counts[k] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); m_mask_left = AVX512_MASK_BITWISE_AND_NOT(m_bin_mask, m_mask_left);//ANDNOT(X, Y) -> NOT X AND Y if(m_mask_left == 0) { break; } else if(k==1){ - counts[0] += bits_set_in_avx512_mask_DOUBLE[m_mask_left]; + counts[0] += AVX512_BIT_COUNT_UNSIGNED_INT(m_mask_left); } } } diff --git a/theory/wp/Makefile b/theory/wp/Makefile index cba558a4..0a6bbc1c 100644 --- a/theory/wp/Makefile +++ b/theory/wp/Makefile @@ -11,7 +11,7 @@ LIBRARY := lib$(LIBNAME).a LIBSRC := countpairs_wp.c countpairs_wp_impl_double.c countpairs_wp_impl_float.c \ $(UTILS_DIR)/gridlink_impl_double.c $(UTILS_DIR)/gridlink_impl_float.c \ $(UTILS_DIR)/gridlink_utils_float.c $(UTILS_DIR)/gridlink_utils_double.c \ - $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c $(UTILS_DIR)/avx512_calls.c + $(UTILS_DIR)/utils.c $(UTILS_DIR)/progressbar.c $(UTILS_DIR)/cpu_features.c LIBRARY_HEADERS := $(LIBNAME).h TARGET := wp diff --git a/theory/wp/wp_kernels.c.src b/theory/wp/wp_kernels.c.src index 06b6d172..2f5de031 100644 --- a/theory/wp/wp_kernels.c.src +++ b/theory/wp/wp_kernels.c.src @@ -23,10 +23,10 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, const weight_struct_DOUBLE *weights0, const int64_t N0, DOUBLE *x1, DOUBLE *y1, DOUBLE *z1, const weight_struct_DOUBLE *weights1, const int64_t N1, const int same_cell, - const DOUBLE sqr_rpmax, const DOUBLE sqr_rpmin, const int nbin, + const DOUBLE sqr_rpmax, const DOUBLE sqr_rpmin, const int nbin, const DOUBLE *rupp_sqr, const DOUBLE pimax, const DOUBLE off_xwrap, const DOUBLE off_ywrap, const DOUBLE off_zwrap, - const DOUBLE min_xdiff, const DOUBLE min_ydiff, const DOUBLE min_zdiff, + const DOUBLE min_xdiff, const DOUBLE min_ydiff, const DOUBLE min_zdiff, const DOUBLE closest_icell_xpos, const DOUBLE closest_icell_ypos, const DOUBLE closest_icell_zpos, DOUBLE *src_rpavg, uint64_t *src_npairs, DOUBLE *src_weightavg, const weight_method_t weight_method) @@ -37,7 +37,7 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 current_utc_time(&tcell_start); uint64_t serial_npairs = 0, vectorized_npairs=0; #endif - + uint64_t npairs[nbin]; for(int i=0;i we can safely terminate the i-loop (i.e., terminate the calculation for this pair of cells) */ @@ -101,8 +101,8 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 i = N0; break; } -#endif - +#endif + /* Now consider if this i'th particle can be a valid pair with ANY of the remaining j' particles. The min. difference in the z-positions between this i'th particle and ANY of the remaining j'th particles, is the difference between the current j'th particle and @@ -115,7 +115,7 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 if(this_dz >= pimax) { continue; } - + /* Okay so there MAY be a pair */ if(same_cell == 1) { z1++; @@ -135,7 +135,7 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 z1++; } } - + /* If no j particle satisfies the constraint for this i particle, then the same holds true for all future i particles because they are sorted in increasing z order (otherwise, we are in the same_cell and there are no further particles) */ @@ -154,17 +154,17 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 const AVX512_FLOATS m_xpos = AVX512_SET_FLOAT(xpos); const AVX512_FLOATS m_ypos = AVX512_SET_FLOAT(ypos); const AVX512_FLOATS m_zpos = AVX512_SET_FLOAT(zpos); - + for(int64_t j=n_off;j= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N1-j]; + AVX512_MASK m_mask_left = (N1 - j) >= AVX512_NVEC ? ~0U:((1U << (unsigned)(N1 - j)) - 1U); const AVX512_FLOATS m_x1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx1); const AVX512_FLOATS m_y1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy1); const AVX512_FLOATS m_z1 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz1); - + union int16 union_rpbin; union float16 union_mDperp; union float16_weights union_mweight; - + union_rpbin.m_ibin = AVX512_SETZERO_INT(); union_mDperp.m_Dperp = AVX512_SETZERO_FLOAT(); @@ -174,29 +174,29 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 localx1 += AVX512_NVEC;//this might actually exceed the allocated range but we will never dereference that localy1 += AVX512_NVEC; localz1 += AVX512_NVEC; - + for(int w = 0; w < pair.num_weights; w++){ pair.weights1[w].a512 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, local_w1.weights[w]); local_w1.weights[w] += AVX512_NVEC; } - + const AVX512_FLOATS m_pimax = AVX512_SET_FLOAT(pimax); const AVX512_FLOATS m_sqr_rpmax = m_rupp_sqr[nbin-1]; const AVX512_FLOATS m_sqr_rpmin = m_rupp_sqr[0]; - + const AVX512_FLOATS m_xdiff = AVX512_SUBTRACT_FLOATS(m_x1, m_xpos); //(x[j] - x0) const AVX512_FLOATS m_ydiff = AVX512_SUBTRACT_FLOATS(m_y1, m_ypos); //(y[j] - y0) const AVX512_FLOATS m_zdiff = AVX512_SUBTRACT_FLOATS(m_z1, m_zpos); //z2[j:j+NVEC-1] - z1 - + if(need_weightavg){ pair.dx.a512 = m_xdiff; pair.dy.a512 = m_ydiff; pair.dz.a512 = m_zdiff; } - + const AVX512_FLOATS m_sqr_xdiff = AVX512_SQUARE_FLOAT(m_xdiff); //(x0 - x[j])^2 const AVX512_FLOATS r2 = AVX512_FMA_ADD_FLOATS(m_ydiff, m_ydiff, m_sqr_xdiff); - + //the z2 arrays are sorted in increasing order. which means //the z2 value will increase in any future iteration of j. //that implies the zdiff values are also monotonically increasing @@ -211,7 +211,7 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 if(m_mask_geq_pimax > 0) { j = N1;//but do not break yet, this chunk might contain valid pairs } - + AVX512_MASK m_mask_pimax = AVX512_MASK_COMPARE_FLOATS(m_mask_left, m_zdiff,m_pimax,_CMP_LT_OS); if(m_mask_pimax == 0) { //None of the dz^2 values satisfies dz^2 < pimax^2 @@ -219,32 +219,32 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 j=N1; break; } - + //Do all the distance cuts using masks here in new scope { const AVX512_MASK m_rpmax_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_pimax, r2, m_sqr_rpmax, _CMP_LT_OQ); - //Create a combined mask + //Create a combined mask //This gives us the mask for all sqr_rpmin <= r2 < sqr_rpmax m_mask_left = AVX512_MASK_COMPARE_FLOATS(m_rpmax_mask, r2, m_sqr_rpmin, _CMP_GE_OQ); if(m_mask_left == 0) { continue; } - + } - + if(need_rpavg) { union_mDperp.m_Dperp = AVX512_MASKZ_SQRT_FLOAT(m_mask_left, r2); } if(need_weightavg){ union_mweight.m_weights = avx512_weight_func(&pair); } - + AVX512_INTS m_rpbin = AVX512_SETZERO_INT(); //Loop backwards through nbins. m_mask_left contains all the points that are less than rpmax - // at the beginning of the loop. + // at the beginning of the loop. for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, r2,m_rupp_sqr[kbin-1],_CMP_GE_OS); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_rpavg || need_weightavg) { m_rpbin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_rpbin, AVX512_SET_INT(kbin)); } @@ -253,9 +253,9 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 break; } }//backwards loop over the bins - + if(need_rpavg || need_weightavg) { - //Do I need this step of going via the union? accessing int[] -> AVX* vector might + //Do I need this step of going via the union? accessing int[] -> AVX* vector might //cause alignment problems but accessing the ints from an AVX* //register should always be fine union_rpbin.m_ibin = m_rpbin; @@ -280,11 +280,11 @@ static inline int wp_avx512_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0 #ifdef COUNT_VECTORIZED uint64_t npairs_found = 0; -#endif +#endif for(int i=0;i we can safely terminate the i-loop (i.e., terminate the calculation for this pair of cells) */ @@ -397,7 +397,7 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, break; } #endif - + /* Now consider if this i'th particle can be a valid pair with ANY of the remaining j' particles. The min. difference in the z-positions between this i'th particle and ANY of the remaining j'th particles, is the difference between the current j'th particle and @@ -410,7 +410,7 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, if(this_dz >= pimax) { continue; } - + /* Okay so there MAY be a pair */ if(same_cell == 1) { z1++; @@ -430,7 +430,7 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, z1++; } } - + /* If no j particle satisfies the constraint for this i particle, then the same holds true for all future i particles because they are sorted in increasing z order (otherwise, we are in the same_cell and there are no further particles) */ @@ -458,27 +458,27 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, const AVX2_FLOATS m_x1 = AVX2_LOAD_FLOATS_UNALIGNED(localx1); const AVX2_FLOATS m_y1 = AVX2_LOAD_FLOATS_UNALIGNED(localy1); const AVX2_FLOATS m_z1 = AVX2_LOAD_FLOATS_UNALIGNED(localz1); - + #ifdef COUNT_VECTORIZED vectorized_npairs += AVX2_NVEC; #endif localx1 += AVX2_NVEC;//this might actually exceed the allocated range but we will never dereference that localy1 += AVX2_NVEC; localz1 += AVX2_NVEC; - + for(int w = 0; w < pair.num_weights; w++){ pair.weights1[w].a = AVX2_LOAD_FLOATS_UNALIGNED(local_w1.weights[w]); local_w1.weights[w] += AVX2_NVEC; } - + const AVX2_FLOATS m_pimax = AVX2_SET_FLOAT(pimax); const AVX2_FLOATS m_sqr_rpmax = m_rupp_sqr[nbin-1]; const AVX2_FLOATS m_sqr_rpmin = m_rupp_sqr[0]; - + const AVX2_FLOATS m_xdiff = AVX2_SUBTRACT_FLOATS(m_x1, m_xpos); //(x[j] - x0) const AVX2_FLOATS m_ydiff = AVX2_SUBTRACT_FLOATS(m_y1, m_ypos); //(y[j] - y0) const AVX2_FLOATS m_zdiff = AVX2_SUBTRACT_FLOATS(m_z1, m_zpos); //z2[j:j+NVEC-1] - z1 - + if(need_weightavg){ pair.dx.a = m_xdiff; pair.dy.a = m_ydiff; @@ -491,7 +491,7 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, const AVX2_FLOATS m_sqr_xdiff = AVX2_SQUARE_FLOAT(m_xdiff); //(x0 - x[j])^2 AVX2_FLOATS r2 = AVX2_FMA_ADD_FLOATS(m_ydiff, m_ydiff, m_sqr_xdiff); - + AVX2_FLOATS m_mask_left; //Do all the distance cuts using masks here in new scope { @@ -505,7 +505,7 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, j=N1;//but do not break, there might be valid pairs in this chunk } - + AVX2_FLOATS m_mask_pimax = AVX2_COMPARE_FLOATS(m_zdiff,m_pimax,_CMP_LT_OS); if(AVX2_TEST_COMPARISON(m_mask_pimax) == 0) { //None of the dz^2 values satisfies dz^2 < pimax^2 @@ -513,25 +513,25 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, j=N1; break; } - + const AVX2_FLOATS m_rpmax_mask = AVX2_COMPARE_FLOATS(r2, m_sqr_rpmax, _CMP_LT_OS); const AVX2_FLOATS m_rpmin_mask = AVX2_COMPARE_FLOATS(r2, m_sqr_rpmin, _CMP_GE_OS); const AVX2_FLOATS m_rp_mask = AVX2_BITWISE_AND(m_rpmax_mask,m_rpmin_mask); - + //Create a combined mask by bitwise and of m1 and m_mask_left. //This gives us the mask for all sqr_rpmin <= r2 < sqr_rpmax m_mask_left = AVX2_BITWISE_AND(m_mask_pimax,m_rp_mask); - + //If not, continue with the next iteration of j-loop const int num_left = AVX2_TEST_COMPARISON(m_mask_left); if(num_left == 0) { continue; } - + //There is some r2 that satisfies sqr_rpmin <= r2 < sqr_rpmax && 0.0 <= dz^2 < pimax^2. r2 = AVX2_BLEND_FLOATS_WITH_MASK(m_sqr_rpmax, r2, m_mask_left); } - + AVX2_FLOATS m_rpbin = AVX2_SETZERO_FLOAT(); if(need_rpavg) { union_mDperp.m_Dperp = AVX2_SQRT_FLOAT(r2); @@ -539,7 +539,7 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, if(need_weightavg){ union_mweight.m_weights = avx_weight_func(&pair); } - + //Loop backwards through nbins. m_mask_left contains all the points that are less than rpmax for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX2_FLOATS m1 = AVX2_COMPARE_FLOATS(r2,m_rupp_sqr[kbin-1],_CMP_GE_OS); @@ -549,14 +549,14 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, if(need_rpavg || need_weightavg) { m_rpbin = AVX2_BLEND_FLOATS_WITH_MASK(m_rpbin,m_kbin[kbin], m_bin_mask); } - + m_mask_left = AVX2_COMPARE_FLOATS(r2,m_rupp_sqr[kbin-1],_CMP_LT_OS); const int test3 = AVX2_TEST_COMPARISON(m_mask_left); if(test3 == 0) { break; } } - + if(need_rpavg || need_weightavg) { union_rpbin.m_ibin = AVX2_TRUNCATE_FLOAT_TO_INT(m_rpbin); //protect the unroll pragma in case compiler is not icc. @@ -575,10 +575,10 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, } } } //OUTPUT_RPAVG - + }//end of j-loop - - //remainder loop + + //remainder loop for(;j= pimax) break; - + const DOUBLE r2 = dx*dx + dy*dy; if(r2 >= sqr_rpmax || r2 < sqr_rpmin) { continue; } - + if(need_weightavg){ pair.dx.d = dx; pair.dy.d = dy; pair.dz.d = dz; } - - + + const DOUBLE r = need_rpavg ? SQRT(r2):ZERO; const DOUBLE pairweight = need_weightavg ? fallback_weight_func(&pair) : ZERO; for(int kbin=nbin-1;kbin>=1;kbin--) { @@ -621,14 +621,14 @@ static inline int wp_avx2_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, } }//remainder loop over second set of particles }//loop over first set of particles - + #ifdef COUNT_VECTORIZED uint64_t npairs_found = 0; -#endif +#endif for(int i=0;i we can safely terminate the i-loop (i.e., terminate the calculation for this pair of cells) */ @@ -742,8 +742,8 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c i = N0; break; } -#endif - +#endif + /* Now consider if this i'th particle can be a valid pair with ANY of the remaining j' particles. The min. difference in the z-positions between this i'th particle and ANY of the remaining j'th particles, is the difference between the current j'th particle and @@ -756,7 +756,7 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c if(this_dz >= pimax) { continue; } - + /* Okay so there MAY be a pair */ if(same_cell == 1) { z1++; @@ -776,7 +776,7 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c z1++; } } - + /* If no j particle satisfies the constraint for this i particle, then the same holds true for all future i particles because they are sorted in increasing z order (otherwise, we are in the same_cell and there are no further particles) */ @@ -784,7 +784,7 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c i = N0; break; } - + int64_t j = z1 - zstart; DOUBLE *localz1 = z1; DOUBLE *localx1 = x1 + j; @@ -792,13 +792,13 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c for(int w = 0; w < local_w1.num_weights; w++){ local_w1.weights[w] = weights1->weights[w] + j; } - + for(;j<=(N1 - AVX_NVEC);j+=AVX_NVEC) { - + const AVX_FLOATS m_xpos = AVX_SET_FLOAT(xpos); const AVX_FLOATS m_ypos = AVX_SET_FLOAT(ypos); const AVX_FLOATS m_zpos = AVX_SET_FLOAT(zpos); - + union int8 union_rpbin; union float8 union_mDperp; union float8_weights union_mweight; @@ -813,20 +813,20 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c localx1 += AVX_NVEC;//this might actually exceed the allocated range but we will never dereference that localy1 += AVX_NVEC; localz1 += AVX_NVEC; - + for(int w = 0; w < pair.num_weights; w++){ pair.weights1[w].a = AVX_LOAD_FLOATS_UNALIGNED(local_w1.weights[w]); local_w1.weights[w] += AVX_NVEC; } - + const AVX_FLOATS m_pimax = AVX_SET_FLOAT(pimax); const AVX_FLOATS m_sqr_rpmax = m_rupp_sqr[nbin-1]; const AVX_FLOATS m_sqr_rpmin = m_rupp_sqr[0]; - + const AVX_FLOATS m_xdiff = AVX_SUBTRACT_FLOATS(m_x1, m_xpos); //(x[j] - x0) const AVX_FLOATS m_ydiff = AVX_SUBTRACT_FLOATS(m_y1, m_ypos); //(y[j] - y0) const AVX_FLOATS m_zdiff = AVX_SUBTRACT_FLOATS(m_z1, m_zpos); //z2[j:j+NVEC-1] - z1 - + if(need_weightavg){ pair.dx.a = m_xdiff; pair.dy.a = m_ydiff; @@ -840,9 +840,9 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c const AVX_FLOATS m_sqr_xdiff = AVX_SQUARE_FLOAT(m_xdiff); //(x0 - x[j])^2 const AVX_FLOATS m_sqr_ydiff = AVX_SQUARE_FLOAT(m_ydiff); //(y0 - y[j])^2 AVX_FLOATS r2 = AVX_ADD_FLOATS(m_sqr_xdiff,m_sqr_ydiff); - + AVX_FLOATS m_mask_left; - + //Do all the distance cuts using masks here in new scope { //the z2 arrays are sorted in increasing order. which means @@ -854,7 +854,7 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c if(AVX_TEST_COMPARISON(m_mask_geq_pimax) > 0) { j=N1;//but do not break yet, this chunk might contain valid pairs } - + AVX_FLOATS m_mask_pimax = AVX_COMPARE_FLOATS(m_zdiff,m_pimax,_CMP_LT_OS); if(AVX_TEST_COMPARISON(m_mask_pimax) == 0) { //None of the dz^2 values satisfies dz^2 < pimax^2 @@ -866,11 +866,11 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c const AVX_FLOATS m_rpmax_mask = AVX_COMPARE_FLOATS(r2, m_sqr_rpmax, _CMP_LT_OS); const AVX_FLOATS m_rpmin_mask = AVX_COMPARE_FLOATS(r2, m_sqr_rpmin, _CMP_GE_OS); const AVX_FLOATS m_rp_mask = AVX_BITWISE_AND(m_rpmax_mask,m_rpmin_mask); - + //Create a combined mask by bitwise and of m1 and m_mask_left. //This gives us the mask for all sqr_rpmin <= r2 < sqr_rpmax m_mask_left = AVX_BITWISE_AND(m_mask_pimax,m_rp_mask); - + //If not, continue with the next iteration of j-loop const int num_left = AVX_TEST_COMPARISON(m_mask_left); if(num_left == 0) { @@ -878,7 +878,7 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c } /* Check if all the possible pairs are in the last bin. But only run - this check if not evaluating same cell pairs or when simply counting + this check if not evaluating same cell pairs or when simply counting the pairs (no rpavg requested) */ if(same_cell == 0 && need_rpavg == 0 && need_weightavg == 0) { const AVX_FLOATS m_last_bin = AVX_BITWISE_AND(m_mask_left, AVX_COMPARE_FLOATS(r2, m_rupp_sqr[nbin-1], _CMP_GE_OS)); @@ -887,7 +887,7 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c continue; } } - + //There is some r2 that satisfies sqr_rpmin <= r2 < sqr_rpmax && 0.0 <= dz^2 < pimax^2. r2 = AVX_BLEND_FLOATS_WITH_MASK(m_sqr_rpmax, r2, m_mask_left); } @@ -899,7 +899,7 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c if(need_weightavg){ union_mweight.m_weights = avx_weight_func(&pair); } - + //Loop backwards through nbins. m_mask_left contains all the points that are less than rpmax for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX_FLOATS m1 = AVX_COMPARE_FLOATS(r2,m_rupp_sqr[kbin-1],_CMP_GE_OS); @@ -915,7 +915,7 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c break; } } - + if(need_rpavg || need_weightavg) { union_rpbin.m_ibin = AVX_TRUNCATE_FLOAT_TO_INT(m_rpbin); //protect the unroll pragma in case compiler is not icc. @@ -936,8 +936,8 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c } //OUTPUT_RPAVG }//end of j-loop - - //remainder loop + + //remainder loop for(;j= pimax) break; - + const DOUBLE r2 = dx*dx + dy*dy; if(r2 >= sqr_rpmax || r2 < sqr_rpmin) { continue; } - + if(need_weightavg){ pair.dx.d = dx; pair.dy.d = dy; @@ -982,11 +982,11 @@ static inline int wp_avx_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c #ifdef COUNT_VECTORIZED uint64_t npairs_found = 0; -#endif +#endif for(int i=0;i we can safely terminate the i-loop (i.e., terminate the calculation for this pair of cells) */ @@ -1100,8 +1100,8 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c i = N0; break; } -#endif - +#endif + /* Now consider if this i'th particle can be a valid pair with ANY of the remaining j' particles. The min. difference in the z-positions between this i'th particle and ANY of the remaining j'th particles, is the difference between the current j'th particle and @@ -1114,7 +1114,7 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c if(this_dz >= pimax) { continue; } - + /* Okay so there MAY be a pair */ if(same_cell == 1) { z1++; @@ -1134,7 +1134,7 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c z1++; } } - + /* If no j particle satisfies the constraint for this i particle, then the same holds true for all future i particles because they are sorted in increasing z order (otherwise, we are in the same_cell and there are no further particles) */ @@ -1142,7 +1142,7 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c i = N0; break; } - + int64_t j = z1 - zstart; DOUBLE *localz1 = z1; DOUBLE *localx1 = x1 + j; @@ -1152,35 +1152,35 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c } for(;j<=(N1 - SSE_NVEC);j+=SSE_NVEC){ - + const SSE_FLOATS m_xpos = SSE_SET_FLOAT(xpos); const SSE_FLOATS m_ypos = SSE_SET_FLOAT(ypos); const SSE_FLOATS m_zpos = SSE_SET_FLOAT(zpos); - + const SSE_FLOATS m_x1 = SSE_LOAD_FLOATS_UNALIGNED(localx1); const SSE_FLOATS m_y1 = SSE_LOAD_FLOATS_UNALIGNED(localy1); const SSE_FLOATS m_z1 = SSE_LOAD_FLOATS_UNALIGNED(localz1); - + #ifdef COUNT_VECTORIZED vectorized_npairs += SSE_NVEC; #endif localx1 += SSE_NVEC; localy1 += SSE_NVEC; localz1 += SSE_NVEC; - + for(int w = 0; w < pair.num_weights; w++){ pair.weights1[w].s = SSE_LOAD_FLOATS_UNALIGNED(local_w1.weights[w]); local_w1.weights[w] += SSE_NVEC; } - + union int4 union_rpbin; union float4 union_mDperp; union float4_weights union_mweight; - + const SSE_FLOATS m_pimax = SSE_SET_FLOAT(pimax); const SSE_FLOATS m_sqr_rpmax = SSE_SET_FLOAT(sqr_rpmax); const SSE_FLOATS m_sqr_rpmin = SSE_SET_FLOAT(sqr_rpmin); - + const SSE_FLOATS m_xdiff = SSE_SUBTRACT_FLOATS(m_x1, m_xpos); //(x[j] - x0) const SSE_FLOATS m_ydiff = SSE_SUBTRACT_FLOATS(m_y1, m_ypos); //(y[j] - y0) const SSE_FLOATS m_zdiff = SSE_SUBTRACT_FLOATS(m_z1, m_zpos); //z2[j:j+NVEC-1] - z1 @@ -1193,10 +1193,10 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c if(SSE_TEST_COMPARISON(m_neg_pimax_mask)==0) { continue; } - + const SSE_FLOATS m_sqr_xdiff = SSE_SQUARE_FLOAT(m_xdiff); const SSE_FLOATS m_sqr_ydiff = SSE_SQUARE_FLOAT(m_ydiff); - + SSE_FLOATS r2 = SSE_ADD_FLOATS(m_sqr_xdiff,m_sqr_ydiff); SSE_FLOATS m_mask_left; { @@ -1223,9 +1223,9 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c if(num_left == 0) { continue; } - + /* Check if all the possible pairs are in the last bin. But only run - this check if not evaluating same cell pairs or when simply counting + this check if not evaluating same cell pairs or when simply counting the pairs (no rpavg requested) */ if(same_cell == 0 && need_rpavg == 0 && need_weightavg == 0) { const SSE_FLOATS m_last_bin = SSE_BITWISE_AND(m_mask_left, SSE_COMPARE_FLOATS_GE(r2, m_rupp_sqr[nbin-1])); @@ -1234,10 +1234,10 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c continue; } } - + r2 = SSE_BLEND_FLOATS_WITH_MASK(m_sqr_rpmax, r2, m_mask_left); } - + SSE_FLOATS m_rpbin = SSE_SETZERO_FLOAT(); if(need_rpavg) { union_mDperp.m_Dperp = SSE_SQRT_FLOAT(r2); @@ -1245,7 +1245,7 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c if(need_weightavg){ union_mweight.m_weights = sse_weight_func(&pair); } - + for(int kbin=nbin-1;kbin>=1;kbin--) { SSE_FLOATS m1 = SSE_COMPARE_FLOATS_GE(r2,m_rupp_sqr[kbin-1]); SSE_FLOATS m_bin_mask = SSE_BITWISE_AND(m1,m_mask_left); @@ -1257,7 +1257,7 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c } if(SSE_TEST_COMPARISON(m_mask_left) == 0) break; } - + if(need_rpavg || need_weightavg) { union_rpbin.m_ibin = SSE_TRUNCATE_FLOAT_TO_INT(m_rpbin); //protect the unroll pragma in case compiler is not icc. @@ -1274,11 +1274,11 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c const DOUBLE weight = union_mweight.weights[jj]; weightavg[kbin] += weight; } - + } } //rpavg - }//j loop over N1, increments of SSE_NVEC - + }//j loop over N1, increments of SSE_NVEC + for(;j= pimax) break; - + const DOUBLE r2 = dx*dx + dy*dy; if(r2 >= sqr_rpmax || r2 < sqr_rpmin) continue; - + if(need_weightavg) { pair.dx.d = dx; pair.dy.d = dy; pair.dz.d = dz; } - + const DOUBLE r = need_rpavg ? SQRT(r2):ZERO; const DOUBLE pairweight = need_weightavg ? fallback_weight_func(&pair) : ZERO; for(int kbin=nbin-1;kbin>=1;kbin--){ @@ -1318,14 +1318,14 @@ static inline int wp_sse_intrinsics_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, c }//searching for kbin loop } } - + #ifdef COUNT_VECTORIZED uint64_t npairs_found = 0; -#endif +#endif for(int i=0;inum_weights; weight_func = get_weight_func_by_method_DOUBLE(weight_method); } - + /* naive implementation that is guaranteed to compile */ const DOUBLE *zstart = z1, *zend = z1 + N1; for(int64_t i=0;iweights[w] + i); } - + #if 0 /* are there any pairs possible with this particle and ANY @@ -1408,7 +1408,7 @@ static inline int wp_fallback_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, const w the current particle (i.e., the smallest remaining z-position among the 'i'-th dataset) and the last j'th particle. Therefore, if this difference is already too large to contain a valid pair, any remaining particle pairs CAN NOT be a valid pair - + --> we can safely terminate the i-loop (i.e., terminate the calculation for this pair of cells) */ @@ -1417,7 +1417,7 @@ static inline int wp_fallback_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, const w break; } #endif - + /* Now consider if this i'th particle can be a valid pair with ANY of the remaining j' particles. The min. difference in the z-positions between this i'th particle and ANY of the remaining j'th particles, is the difference between the current j'th particle and @@ -1430,7 +1430,7 @@ static inline int wp_fallback_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, const w if(this_dz >= pimax) { continue; } - + /* Okay so there MAY be a pair */ if(same_cell == 1) { z1++; @@ -1450,7 +1450,7 @@ static inline int wp_fallback_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, const w z1++; } } - + /* If no j particle satisfies the constraint for this i particle, then the same holds true for all future i particles because they are sorted in increasing z order (otherwise, we are in the same_cell and there are no further particles) */ @@ -1458,13 +1458,13 @@ static inline int wp_fallback_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, const w i = N0; break; } - + const int64_t n_off = z1 - zstart; const int64_t nleft = N1 - n_off; DOUBLE *localz1 = z1; DOUBLE *localx1 = x1 + n_off; DOUBLE *localy1 = y1 + n_off; - + for(int64_t j=0;j= pimax) break; - + const DOUBLE r2 = dx*dx + dy*dy; if(r2 >= sqr_rpmax || r2 < sqr_rpmin) continue; @@ -1508,11 +1508,11 @@ static inline int wp_fallback_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, const w #ifdef COUNT_VECTORIZED uint64_t npairs_found = 0; -#endif +#endif for(int i=0;i= AVX512_NVEC ? ~0:masks_per_misalignment_value_DOUBLE[N2-j]; + AVX512_MASK m_mask_left = (N2 - j) >= AVX512_NVEC ? ~0U:masks_per_misalignment_value_DOUBLE[N2-j]; const AVX512_FLOATS m_x2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localx2); const AVX512_FLOATS m_y2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localy2); const AVX512_FLOATS m_z2 = AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(m_mask_left, localz2); @@ -218,7 +218,7 @@ static inline int xi_avx512_intrinsics_DOUBLE(DOUBLE *x1, DOUBLE *y1, DOUBLE *z1 // at the beginning of the loop. for(int kbin=nbin-1;kbin>=1;kbin--) { const AVX512_MASK m_bin_mask = AVX512_MASK_COMPARE_FLOATS(m_mask_left, r2,m_rupp_sqr[kbin-1],_CMP_GE_OQ); - npairs[kbin] += bits_set_in_avx512_mask_DOUBLE[m_bin_mask]; + npairs[kbin] += AVX512_BIT_COUNT_UNSIGNED_INT(m_bin_mask); if(need_rpavg || need_weightavg) { m_rpbin = AVX512_BLEND_INTS_WITH_MASK(m_bin_mask, m_rpbin, AVX512_SET_INT(kbin)); } @@ -939,7 +939,7 @@ static inline int xi_fallback_DOUBLE(DOUBLE *x0, DOUBLE *y0, DOUBLE *z0, const w const DOUBLE r2 = dx*dx + dy*dy + dz*dz; if(r2 >= sqr_rmax || r2 < sqr_rmin) continue; - DOUBLE pairweight = ZERO; + DOUBLE pairweight = ZERO; if(src_weightavg != NULL){ for(int w = 0; w < pair.num_weights; w++){ pair.weights1[w].d = *(weights1->weights[w] + n_off + j); diff --git a/utils/avx512_calls.c b/utils/avx512_calls.c deleted file mode 100644 index 6f3f4f6e..00000000 --- a/utils/avx512_calls.c +++ /dev/null @@ -1,40 +0,0 @@ -/* File: avx512_calls.c */ -/* - This file is a part of the Corrfunc package - Copyright (C) 2015-- Manodeep Sinha (manodeep@gmail.com) - License: MIT LICENSE. See LICENSE file under the top-level - directory at https://github.com/manodeep/Corrfunc/ -*/ -#if defined(__AVX512F__) -#include "avx512_calls.h" - - -const int64_t bits_set_in_avx512_mask_float[] = { B16(0) }; -const uint16_t masks_per_misalignment_value_float[] = {0b1111111111111111, - 0b0000000000000001, - 0b0000000000000011, - 0b0000000000000111, - 0b0000000000001111, - 0b0000000000011111, - 0b0000000000111111, - 0b0000000001111111, - 0b0000000011111111, - 0b0000000111111111, - 0b0000001111111111, - 0b0000011111111111, - 0b0000111111111111, - 0b0001111111111111, - 0b0011111111111111, - 0b0111111111111111}; - -const int64_t bits_set_in_avx512_mask_double[] = { B8(0) }; -const uint8_t masks_per_misalignment_value_double[] = {0b11111111, - 0b00000001, - 0b00000011, - 0b00000111, - 0b00001111, - 0b00011111, - 0b00111111, - 0b01111111}; - -#endif diff --git a/utils/avx512_calls.h b/utils/avx512_calls.h index 5ff60f73..511752b5 100644 --- a/utils/avx512_calls.h +++ b/utils/avx512_calls.h @@ -19,7 +19,7 @@ extern "C" { #endif -#include "function_precision.h" +#include "function_precision.h" #define PREFETCH(mem) asm ("prefetcht0 %0"::"m"(mem)) @@ -41,23 +41,11 @@ extern "C" { #define AVX512_MASK_BITWISE_AND_NOT(X,Y) _mm512_kandn(X,Y) //~X & Y #define AVX512_MASK_BITWISE_NOT(X,Y) _mm512_knot(X) //~X - - - /* For setting up the array that contains the number bits set in a mask */ - // Can be used to generate up to 16 bit lookup tables -# define B2(n) n, n+1, n+1, n+2 -# define B4(n) B2(n), B2(n+1), B2(n+1), B2(n+2) -# define B6(n) B4(n), B4(n+1), B4(n+1), B4(n+2) -# define B8(n) B6(n), B6(n+1), B6(n+1), B6(n+2) -# define B10(n) B8(n), B8(n+1), B8(n+1), B8(n+2) -# define B12(n) B10(n), B10(n+1), B10(n+1), B10(n+2) -# define B14(n) B12(n), B12(n+1), B12(n+1), B12(n+2) -# define B16(n) B14(0),B14(1), B14(1), B14(2) #ifndef DOUBLE_PREC -#define DOUBLE float -#define AVX512_NVEC 16 +#define DOUBLE float +#define AVX512_NVEC 16 #define AVX512_MASK __mmask16 #define AVX512_FLOATS __m512 @@ -65,21 +53,21 @@ extern "C" { #define AVX512_SET_INT(X) _mm512_set1_epi32(X) #define AVX512_SETZERO_INT() _mm512_setzero_epi32() - + #if 0 -/* commenting out the integer math operations since they are either cumbersome or produce results of different SIMD widths*/ -#define AVX512_ADD_INTS(X, Y) _mm512_add_epi32(X, Y) +/* commenting out the integer math operations since they are either cumbersome or produce results of different SIMD widths*/ +#define AVX512_ADD_INTS(X, Y) _mm512_add_epi32(X, Y) #define AVX512_MASK_ADD_INTS(FALSEVALS, MASK, X, Y) _mm512_mask_add_epi32(FALSEVALS, MASK, X, Y) #define AVX512_MASKZ_ADD_INTS(MASK, X, Y) _mm512_maskz_add_epi32(MASK, X, Y) -#define AVX512_MULTIPLY_INTS(X, Y) _mm512_mul_epi32(X, Y) +#define AVX512_MULTIPLY_INTS(X, Y) _mm512_mul_epi32(X, Y) #define AVX512_MASK_MULTIPLY_INTS(FALSEVALS, MASK, X, Y) _mm512_mask_mul_epi32(FALSEVALS, MASK, X, Y) #define AVX512_MASKZ_MULTIPLY_INTS(MASK, X, Y) _mm512_maskz_mul_epi32(MASK, X, Y) #define AVX512_MASKZ_MULTIPLY_INTS_LOW32(MASK, X, Y) _mm512_maskz_mullo_epi32(MASK, X, Y) #endif /*end of integer math*/ #define AVX512_SETZERO_FLOAT() _mm512_setzero_ps() - + #define AVX512_LOAD_FLOATS_UNALIGNED(X) _mm512_loadu_ps(X) #define AVX512_MASK_LOAD_FLOATS_UNALIGNED(FALSEVALS, MASK, X) _mm512_mask_loadu_ps(FALSEVALS, MASK, X) #define AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(MASK, X) _mm512_maskz_loadu_ps(MASK, X) @@ -112,9 +100,9 @@ extern "C" { #define AVX512_MASKZ_FMA_ADD_FLOATS(X, MASK, Y, Z) _mm512_maskz_fmadd_ps(MASK, X, Y, Z) /* returns Z - XY*/ -#define AVX512_FNMA_ADD_FLOATS(X, Y, Z) _mm512_fnmadd_ps(X, Y, Z) -#define AVX512_MASK_FNMA_ADD_FLOATS(X, MASK, Y, Z) _mm512_mask_fnmadd_ps(X, MASK, Y, Z) -#define AVX512_MASKZ_FNMA_ADD_FLOATS(X, MASK, Y, Z) _mm512_maskz_fnmadd_ps(MASK, X, Y, Z) +#define AVX512_FNMA_ADD_FLOATS(X, Y, Z) _mm512_fnmadd_ps(X, Y, Z) +#define AVX512_MASK_FNMA_ADD_FLOATS(X, MASK, Y, Z) _mm512_mask_fnmadd_ps(X, MASK, Y, Z) +#define AVX512_MASKZ_FNMA_ADD_FLOATS(X, MASK, Y, Z) _mm512_maskz_fnmadd_ps(MASK, X, Y, Z) /* returns XY - Z */ #define AVX512_FMA_SUBTRACT_FLOATS(X,Y,Z) _mm512_fmsub_ps(X,Y,Z) @@ -161,7 +149,7 @@ extern "C" { //Trig #ifdef __INTEL_COMPILER /* Needs SVML */ -#define AVX512_ARC_COSINE(X, order) _mm512_acos_ps(X) +#define AVX512_ARC_COSINE(X, order) _mm512_acos_ps(X) #else //Other compilers do not have the vectorized arc-cosine #define AVX512_ARC_COSINE(X, order) inv_cosine_avx512(X, order) @@ -173,28 +161,28 @@ extern "C" { //Absolute value #define AVX512_ABS_FLOAT(X) _mm512_abs_ps(X) - + //Casting (does not actual convert between types) #define AVX512_CAST_FLOAT_TO_INT(X) _mm512_castps_si512(X) #define AVX512_CAST_INT_TO_FLOAT(X) _mm512_castsi512_ps(X) #else //DOUBLE PRECISION CALCULATIONS - + #define DOUBLE double -#define AVX512_NVEC 8 +#define AVX512_NVEC 8 #define AVX512_MASK __mmask8 #define AVX512_FLOATS __m512d -//This is AVX2 and not AVX512F +//This is AVX2 and not AVX512F #define AVX512_INTS __m256i #define AVX512_SET_INT(X) _mm256_set1_epi32(X) #define AVX512_SETZERO_INT() _mm256_setzero_si256() - + #if 0 -#define AVX512_ADD_INTS(X, Y) _mm256_add_epi32(X, Y) -#define AVX512_MULTIPLY_INTS(X, Y) _mm256_mul_epi32(X, Y) +#define AVX512_ADD_INTS(X, Y) _mm256_add_epi32(X, Y) +#define AVX512_MULTIPLY_INTS(X, Y) _mm256_mul_epi32(X, Y) #if defined(__AVX512VL__) #define AVX512_MASK_ADD_INTS(FALSEVALS, MASK, X, Y) _mm256_mask_add_epi32(FALSEVALS, MASK, X, Y) @@ -203,7 +191,7 @@ extern "C" { #define AVX512_MASK_MULTIPLY_INTS(FALSEVALS, MASK, X, Y) _mm256_mask_mul_epi32(FALSEVALS, MASK, X, Y) #define AVX512_MASKZ_MULTIPLY_INTS(MASK, X, Y) _mm256_maskz_mul_epi32(MASK, X, Y) #define AVX512_MASKZ_MULTIPLY_INTS_LOW32(MASK, X, Y) _mm256_maskz_mullo_epi32(MASK, X, Y) - + #elif defined(__AVX512F__) #define AVX512_MASK_ADD_INTS(FALSEVALS, MASK, X, Y) _mm512_castsi512_si256(_mm512_mask_add_epi32(_mm512_castsi256_si512(FALSEVALS), MASK, _mm512_castsi256_si512(X), _mm512_castsi256_si512(Y))) #define AVX512_MASKZ_ADD_INTS(MASK, X, Y) _mm512_castsi512_si256(_mm512_maskz_add_epi32(MASK, _mm512_castsi256_si512(X), _mm512_castsi256_si512(Y))) @@ -217,7 +205,7 @@ extern "C" { #endif /* commenting out the int math operations since they are either cumbersome or produce results of different SIMD widths*/ #define AVX512_SETZERO_FLOAT() _mm512_setzero_pd() - + #define AVX512_LOAD_FLOATS_UNALIGNED(X) _mm512_loadu_pd(X) #define AVX512_MASK_LOAD_FLOATS_UNALIGNED(FALSEVALS, MASK, X) _mm512_mask_loadu_pd(FALSEVALS, MASK, X) #define AVX512_MASKZ_LOAD_FLOATS_UNALIGNED(MASK, X) _mm512_maskz_loadu_pd(MASK, X) @@ -250,9 +238,9 @@ extern "C" { #define AVX512_MASKZ_FMA_ADD_FLOATS(X, MASK, Y, Z) _mm512_maskz_fmadd_pd(MASK, X, Y, Z) /* returns Z - XY*/ -#define AVX512_FNMA_ADD_FLOATS(X, Y, Z) _mm512_fnmadd_pd(X, Y, Z) -#define AVX512_MASK_FNMA_ADD_FLOATS(X, MASK, Y, Z) _mm512_mask_fnmadd_pd(X, MASK, Y, Z) -#define AVX512_MASKZ_FNMA_ADD_FLOATS(X, MASK, Y, Z) _mm512_maskz_fnmadd_pd(MASK, X, Y, Z) +#define AVX512_FNMA_ADD_FLOATS(X, Y, Z) _mm512_fnmadd_pd(X, Y, Z) +#define AVX512_MASK_FNMA_ADD_FLOATS(X, MASK, Y, Z) _mm512_mask_fnmadd_pd(X, MASK, Y, Z) +#define AVX512_MASKZ_FNMA_ADD_FLOATS(X, MASK, Y, Z) _mm512_maskz_fnmadd_pd(MASK, X, Y, Z) /* returns XY - Z */ #define AVX512_FMA_SUBTRACT_FLOATS(X,Y,Z) _mm512_fmsub_pd(X,Y,Z) @@ -271,7 +259,7 @@ extern "C" { #define AVX512_SVML_SQRT_FLOAT(X) _mm512_svml_sqrt_pd(X) -#define AVX512_TRUNCATE_FLOAT_TO_INT(X) _mm512_cvttpd_epi32(X) +#define AVX512_TRUNCATE_FLOAT_TO_INT(X) _mm512_cvttpd_epi32(X) #define AVX512_STORE_FLOATS_TO_MEMORY(X,Y) _mm512_storeu_pd(X,Y) #define AVX512_SQUARE_FLOAT(X) _mm512_mul_pd(X,X) #define AVX512_MASKZ_SQUARE_FLOAT(MASK, X) _mm512_maskz_mul_pd(MASK, X,X) @@ -312,7 +300,7 @@ extern "C" { #define AVX512_MAX_FLOATS(X,Y) _mm512_max_pd(X,Y) //Absolute value -#if __GNUC__ <= 8 +#if __GNUC__ <= 8 //there was a bug for the function proto-type //for _mm512_abs_pd -- see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87467 //Need to protect the user @@ -320,7 +308,7 @@ extern "C" { #else #define AVX512_ABS_FLOAT(X) _mm512_abs_pd(X) #endif - + //Casting (does not actual convert between types) #define AVX512_CAST_FLOAT_TO_INT(X) _mm512_castpd_si512(X) #define AVX512_CAST_INT_TO_FLOAT(X) _mm512_castsi512_pd(_mm512_castsi256_si512(X)) @@ -329,7 +317,7 @@ extern "C" { #ifndef __INTEL_COMPILER #include "fast_acos.h" - + static inline AVX512_FLOATS inv_cosine_avx512(const AVX512_FLOATS X, const int order) { union cos{ @@ -348,7 +336,7 @@ static inline AVX512_FLOATS inv_cosine_avx512(const AVX512_FLOATS X, const int o union_costheta.x[ii] = costheta <= minus_one ? minus_one:costheta; union_costheta.x[ii] = costheta >= one ? one:costheta; } - + if(order == 0) { for(int ii=0;ii