Skip to content

Refactor/perf batch 2 #661

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
timcadman wants to merge 11 commits into
base: v7.0-dev
Choose a base branch
from
Open

Refactor/perf batch 2 #661

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
db931b2
chore: bring in utils file from previous PR
timcadman Apr 13, 2026
d79d871
refactor: move checks to serverside
timcadman Apr 13, 2026
fa381c1
docs: ran document
timcadman Apr 13, 2026
2aa71ad
test: updated tests following refactor
timcadman Apr 13, 2026
d4eabb7
refactor: add back in class consistency checks
timcadman Apr 13, 2026
01d7066
refactor: don't return validity message
timcadman Apr 13, 2026
8d621f8
update dsBase testing tar
timcadman Apr 13, 2026
f2cb293
test: updated expectations with new return value
timcadman Apr 13, 2026
1b921ea
refactor: add classConsistencyCheck parameter, remove from ds.levels
timcadman Apr 13, 2026
a6d9f06
test: add performance tests for batch 2 functions
timcadman Apr 13, 2026
4db816e
docs: update roxygen for classConsistencyCheck, remove stale checks p…
timcadman Apr 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions NAMESPACE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ export(ds.var)
export(ds.vectorCalc)
import(DSI)
import(data.table)
importFrom(DSI,datashield.connections_find)
importFrom(cli,cli_abort)
importFrom(stats,as.formula)
importFrom(stats,na.omit)
importFrom(stats,ts)
Expand Down
14 changes: 2 additions & 12 deletions R/ds.class.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
#' @return \code{ds.class} returns the type of the R object.
#' @author DataSHIELD Development Team
#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @seealso \code{\link{ds.exists}} to verify if an object is defined (exists) on the server-side.
#' @examples
#' \dontrun{
Expand Down Expand Up @@ -54,23 +55,12 @@
#'
ds.class <- function(x=NULL, datasources=NULL) {

# look for DS connections
if(is.null(datasources)){
datasources <- datashield.connections_find()
}

# ensure datasources is a list of DSConnection-class
if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
}
datasources <- .set_datasources(datasources)

if(is.null(x)){
stop("Please provide the name of the input object!", call.=FALSE)
}

# check if the input object is defined in all the studies
defined <- isDefined(datasources, x)

cally <- call('classDS', x)
output <- DSI::datashield.aggregate(datasources, cally)

Expand Down
107 changes: 3 additions & 104 deletions R/ds.completeCases.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,123 +68,22 @@
#' }
#'
#' @author DataSHIELD Development Team
#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
#'
ds.completeCases <- function(x1=NULL, newobj=NULL, datasources=NULL){

# if no connection login details are provided look for 'connection' objects in the environment
if(is.null(datasources)){
datasources <- datashield.connections_find()
}

# ensure datasources is a list of DSConnection-class
if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
}
datasources <- .set_datasources(datasources)

# check if a value has been provided for x1
if(is.null(x1)){
return("Error: x1 must be a character string naming a serverside data.frame, matrix or vector")
}

# check if the input object is defined in all the studies
isDefined(datasources, x1)

# rename target object for transfer (not strictly necessary as string will pass parser anyway)
# but maintains consistency with other functions
x1.transmit <- x1

# if no value specified for output object, then specify a default
if(is.null(newobj)){
newobj <- paste0(x1,"_complete.cases")
}

# CALL THE MAIN SERVER SIDE FUNCTION
calltext <- call("completeCasesDS", x1.transmit)
calltext <- call("completeCasesDS", x1)
DSI::datashield.assign(datasources, newobj, calltext)


#############################################################################################################
#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
#
#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
test.obj.name<-newobj #
#
#TRACER #
#return(test.obj.name) #
#} #
#
#
# CALL SEVERSIDE FUNCTION #
calltext <- call("testObjExistsDS", test.obj.name) #
#
object.info<-DSI::datashield.aggregate(datasources, calltext) #
#
# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
num.datasources<-length(object.info) #
#
#
obj.name.exists.in.all.sources<-TRUE #
obj.non.null.in.all.sources<-TRUE #
#
for(j in 1:num.datasources){ #
if(!object.info[[j]]$test.obj.exists){ #
obj.name.exists.in.all.sources<-FALSE #
} #
if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
obj.non.null.in.all.sources<-FALSE #
} #
} #
#
if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
#
return.message<- #
paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
#
#
}else{ #
#
return.message.1<- #
paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
#
return.message.2<- #
paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
#
return.message.3<- #
paste0("Please use ds.ls() to identify where missing") #
#
#
return.message<-list(return.message.1,return.message.2,return.message.3) #
#
} #
#
calltext <- call("messageDS", test.obj.name) #
studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
#
no.errors<-TRUE #
for(nd in 1:num.datasources){ #
if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
no.errors<-FALSE #
} #
} #
#
#
if(no.errors){ #
validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
return(list(is.object.created=return.message,validity.check=validity.check)) #
} #
#
if(!no.errors){ #
validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
return(list(is.object.created=return.message,validity.check=validity.check, #
studyside.messages=studyside.message)) #
} #
#
#END OF CHECK OBJECT CREATED CORECTLY MODULE #
#############################################################################################################

}
#ds.completeCases


70 changes: 22 additions & 48 deletions R/ds.dim.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,28 +7,26 @@
#' from every single study and the pooled dimension of the object by summing up the individual
#' dimensions returned from each study.
#'
#' In \code{checks} parameter is suggested that checks should only be undertaken once the
#' function call has failed.
#'
#' Server function called: \code{dimDS}
#'
#' @param x a character string providing the name of the input object.
#' @param type a character string that represents the type of analysis to carry out.
#'
#' @param x a character string providing the name of the input object.
#' @param type a character string that represents the type of analysis to carry out.
#' If \code{type} is set to \code{'combine'}, \code{'combined'}, \code{'combines'} or \code{'c'},
#' the global dimension is returned.
#' If \code{type} is set to \code{'split'}, \code{'splits'} or \code{'s'},
#' the global dimension is returned.
#' If \code{type} is set to \code{'split'}, \code{'splits'} or \code{'s'},
#' the dimension is returned separately for each study.
#' If \code{type} is set to \code{'both'} or \code{'b'}, both sets of outputs are produced.
#' Default \code{'both'}.
#' @param checks logical. If TRUE undertakes all DataSHIELD checks (time-consuming).
#' Default FALSE.
#' Default \code{'both'}.
#' @param classConsistencyCheck logical. If TRUE, checks that the input object has the same
#' class across all studies. Default TRUE.
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}}
#' objects obtained after login. If the \code{datasources} argument is not specified
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
#' @return \code{ds.dim} retrieves to the client-side the dimension of the object
#' in the form of a vector where the first
#' element indicates the number of rows and the second element indicates the number of columns.
#' @author DataSHIELD Development Team
#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @seealso \code{\link{ds.dataFrame}} to generate a table of the type data frame.
#' @seealso \code{\link{ds.changeRefGroup}} to change the reference level of a factor.
#' @seealso \code{\link{ds.colnames}} to obtain the column names of a matrix or a data frame
Expand Down Expand Up @@ -67,68 +65,44 @@
#' # Calculate the dimension
#' ds.dim(x="D",
#' type="combine", #global dimension
#' checks = FALSE,
#' datasources = connections)#all opal servers are used
#'#' datasources = connections)#all opal servers are used
#' ds.dim(x="D",
#' type = "both",#separate dimension for each study
#' #and the pooled dimension (default)
#' checks = FALSE,
#' datasources = connections)#all opal servers are used
#'#' datasources = connections)#all opal servers are used
#' ds.dim(x="D",
#' type="split", #separate dimension for each study
#' checks = FALSE,
#' datasources = connections[1])#only the first opal server is used ("study1")
#'#' datasources = connections[1])#only the first opal server is used ("study1")
#'
#' # clear the Datashield R sessions and logout
#' datashield.logout(connections)
#'
#' }
#'
ds.dim <- function(x=NULL, type='both', checks=FALSE, datasources=NULL) {
ds.dim <- function(x=NULL, type='both', classConsistencyCheck=TRUE, datasources=NULL) {

# look for DS connections
if(is.null(datasources)){
datasources <- datashield.connections_find()
}

# ensure datasources is a list of DSConnection-class
if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
}
datasources <- .set_datasources(datasources)

if(is.null(x)){
stop("Please provide the name of a data.frame or matrix!", call.=FALSE)
}

########################################################################################################
# MODULE: GENERIC OPTIONAL CHECKS TO ENSURE CONSISTENT STRUCTURE OF KEY VARIABLES IN DIFFERENT SOURCES #
# beginning of optional checks - the process stops and reports as soon as one check fails #
# #
if(checks){ #
message(" -- Verifying the variables in the model") #
# check if the input object(s) is(are) defined in all the studies #
defined <- isDefined(datasources, x) # #
# call the internal function that checks the input object is suitable in all studies #
typ <- checkClass(datasources, x) #
# throw a message and stop if input is not table structure #
if(!('data.frame' %in% typ) & !('matrix' %in% typ)){ #
stop("The input object must be a table structure!", call.=FALSE) #
} #
} #
########################################################################################################


###################################################################################################
#MODULE: EXTEND "type" argument to include "both" and enable valid aliases #
if(type == 'combine' | type == 'combined' | type == 'combines' | type == 'c') type <- 'combine' #
if(type == 'split' | type == 'splits' | type == 's') type <- 'split' #
if(type == 'both' | type == 'b' ) type <- 'both' #
#
#MODIFY FUNCTION CODE TO DEAL WITH ALL THREE TYPES #
###################################################################################################

cally <- call("dimDS", x)
dimensions <- DSI::datashield.aggregate(datasources, cally)
results <- DSI::datashield.aggregate(datasources, cally)

if(classConsistencyCheck){
.checkClassConsistency(results)
}

# extract dimensions from results
dimensions <- lapply(results, function(r) r$dim)

# names of the studies to be used in the output
stdnames <- names(datasources)
Expand Down
44 changes: 13 additions & 31 deletions R/ds.isNA.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#' @return \code{ds.isNA} returns a boolean. If it is TRUE the vector is empty
#' (all values are NA), FALSE otherwise.
#' @author DataSHIELD Development Team
#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
#' @export
#' @examples
#' \dontrun{
Expand Down Expand Up @@ -55,48 +56,29 @@
#'
#' }
#'
ds.isNA <- function(x=NULL, datasources=NULL){
ds.isNA <- function(x=NULL, classConsistencyCheck=TRUE, datasources=NULL){

# look for DS connections
if(is.null(datasources)){
datasources <- datashield.connections_find()
}

# ensure datasources is a list of DSConnection-class
if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
}
datasources <- .set_datasources(datasources)

if(is.null(x)){
stop("Please provide the name of the input vector!", call.=FALSE)
}

# check if the input object is defined in all the studies
isDefined(datasources, x)

# call the internal function that checks the input object is of the same class in all studies.
typ <- checkClass(datasources, x)

# the input object must be a vector
if(!('character' %in% typ) & !('factor' %in% typ) & !('integer' %in% typ) & !('logical' %in% typ) & !('numeric' %in% typ) & !('data.frame' %in% typ) & !('matrix' %in% typ)){
stop("The input object must be a character, factor, integer, logical or numeric vector.", call.=FALSE)
}

# name of the studies to be used in the plots' titles
stdnames <- names(datasources)

# name of the variable
xnames <- extract(x)
varname <- xnames$elements

# keep of the results of the checks for each study
track <- list()
cally <- call("isNaDS", x)
results <- DSI::datashield.aggregate(datasources, cally)

# call server side function 'isNaDS' to check, in each study, if the vector is empty
for(i in 1: length(datasources)){
cally <- call("isNaDS", x)
out <- DSI::datashield.aggregate(datasources[i], cally)
if(out[[1]]){
if(classConsistencyCheck){
.checkClassConsistency(results)
}

# report per-study if all NA
track <- list()
for(i in 1:length(results)){
if(results[[i]]$is.na){
track[[i]] <- TRUE
message("The variable ", varname, " in ", stdnames[i], " is missing at complete (all values are 'NA').")
}else{
Expand Down
Loading
Loading