diff --git a/exist-core/pom.xml b/exist-core/pom.xml index 78b71881e2..a7974642c9 100644 --- a/exist-core/pom.xml +++ b/exist-core/pom.xml @@ -1015,12 +1015,15 @@ src/test/java/org/exist/security/internal/BackupRestoreSecurityPrincipalsTest.java src/main/java/org/exist/security/internal/RealmImpl.java src/main/java/org/exist/security/internal/SecurityManagerImpl.java + src/main/java/org/exist/security/internal/SMEvents.java src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java src/main/java/org/exist/security/management/AccountsManagement.java src/main/java/org/exist/security/management/GroupsManagement.java src/main/java/org/exist/source/AbstractSource.java + src/main/java/org/exist/source/DBSource.java src/main/java/org/exist/source/Source.java src/main/java/org/exist/source/SourceFactory.java + src/test/java/org/exist/source/SourceFactoryTest.java src/main/java/org/exist/source/URLSource.java src/test/java/org/exist/stax/EmbeddedXMLStreamReaderTest.java src/test/java/org/exist/storage/AbstractUpdateTest.java @@ -1088,6 +1091,8 @@ src/test/resources-filtered/org/exist/storage/statistics/conf.xml src/main/java/org/exist/storage/sync/SyncTask.java src/test/java/org/exist/storage/util/PauseFunction.java + src/main/java/org/exist/test/ExistEmbeddedServer.java + src/main/java/org/exist/test/ExistWebServer.java src/main/java/org/exist/test/ExistXmldbEmbeddedServer.java src/main/java/org/exist/test/TransactionTestDSL.java src/main/java/org/exist/test/XQueryAssertions.java @@ -1232,11 +1237,15 @@ src/test/java/org/exist/xmldb/concurrent/action/ValueAppendAction.java src/test/java/org/exist/xmldb/concurrent/action/XQueryAction.java src/test/java/org/exist/xmldb/concurrent/action/XQueryUpdateAction.java + src/main/java/org/exist/xmlrpc/AbstractCachedResult.java + src/main/java/org/exist/xmlrpc/CachedQueryResult.java src/main/java/org/exist/xmlrpc/ExistRpcTypeFactory.java src/test/java/org/exist/xmlrpc/MimeTypeTest.java + src/main/java/org/exist/xmlrpc/QueryResultCache.java src/test/java/org/exist/xmlrpc/QuerySessionTest.java src/main/java/org/exist/xmlrpc/RpcAPI.java src/main/java/org/exist/xmlrpc/RpcConnection.java + src/main/java/org/exist/xmlrpc/XmldbRequestProcessorFactory.java src/test/java/org/exist/xmlrpc/XmlRpcTest.java src/main/java/org/exist/xqj/Marshaller.java src/test/java/org/exist/xqj/MarshallerTest.java @@ -1248,6 +1257,7 @@ src/test/java/org/exist/xquery/CardinalityTest.java src/test/java/org/exist/xquery/CleanupTest.java src/main/java/org/exist/xquery/CombiningExpression.java + src/main/java/org/exist/xquery/CompiledXQuery.java src/test/java/org/exist/xquery/ConstructedNodesRecoveryTest.java src/test/java/org/exist/xquery/ConstructedNodesTest.java src/main/java/org/exist/xquery/Context.java @@ -1465,6 +1475,7 @@ src/test/java/org/exist/xquery/functions/util/Base64FunctionsTest.java src/test/java/org/exist/xquery/functions/util/BaseConverterTest.java src/main/java/org/exist/xquery/functions/util/BuiltinFunctions.java + src/main/java/org/exist/xquery/functions/util/CollectionName.java src/main/java/org/exist/xquery/functions/util/DescribeFunction.java src/test/java/org/exist/xquery/functions/util/EvalTest.java src/test/java/org/exist/xquery/functions/util/ExpandTest.java @@ -1841,12 +1852,18 @@ src/test/java/org/exist/security/internal/BackupRestoreSecurityPrincipalsTest.java src/main/java/org/exist/security/internal/RealmImpl.java src/main/java/org/exist/security/internal/SecurityManagerImpl.java + src/main/java/org/exist/security/internal/SMEvents.java + src/main/java/org/exist/security/internal/aider/ImmutableUnixStylePermissionAider.java src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java src/main/java/org/exist/security/management/AccountsManagement.java src/main/java/org/exist/security/management/GroupsManagement.java src/main/java/org/exist/source/AbstractSource.java + src/main/java/org/exist/source/DBSource.java + src/main/java/org/exist/source/DbStoreSource.java + src/main/java/org/exist/source/DbUriSource.java src/main/java/org/exist/source/Source.java src/main/java/org/exist/source/SourceFactory.java + src/test/java/org/exist/source/SourceFactoryTest.java src/main/java/org/exist/source/URLSource.java src/test/java/org/exist/stax/EmbeddedXMLStreamReaderTest.java src/test/java/org/exist/storage/AbstractRecoverTest.java @@ -1966,6 +1983,8 @@ src/test/java/org/exist/storage/txn/TxnTest.java src/test/java/org/exist/storage/util/PauseFunction.java src/main/java/org/exist/test/DiffMatcher.java + src/main/java/org/exist/test/ExistEmbeddedServer.java + src/main/java/org/exist/test/ExistWebServer.java src/main/java/org/exist/test/ExistXmldbEmbeddedServer.java src/main/java/org/exist/test/TransactionTestDSL.java src/test/java/org/exist/test/Util.java @@ -2072,6 +2091,7 @@ src/test/java/org/exist/xmldb/IndexingTest.java src/main/java/org/exist/xmldb/LocalBinaryResource.java src/main/java/org/exist/xmldb/LocalCollection.java + src/main/java/org/exist/xmldb/LocalCompiledExpression.java src/main/java/org/exist/xmldb/LocalResourceSet.java src/main/java/org/exist/xmldb/LocalRestoreService.java src/main/java/org/exist/xmldb/LocalXMLResource.java @@ -2124,15 +2144,19 @@ src/test/java/org/exist/xmldb/concurrent/action/ValueAppendAction.java src/test/java/org/exist/xmldb/concurrent/action/XQueryAction.java src/test/java/org/exist/xmldb/concurrent/action/XQueryUpdateAction.java + src/main/java/org/exist/xmlrpc/AbstractCachedResult.java src/main/java/org/exist/xmlrpc/ACEAiderParser.java src/main/java/org/exist/xmlrpc/ACEAiderSerializer.java src/main/java/org/exist/xmlrpc/ArrayWrapperParser.java src/main/java/org/exist/xmlrpc/ArrayWrapperSerializer.java + src/main/java/org/exist/xmlrpc/CachedQueryResult.java src/main/java/org/exist/xmlrpc/ExistRpcTypeFactory.java src/test/java/org/exist/xmlrpc/MimeTypeTest.java + src/main/java/org/exist/xmlrpc/QueryResultCache.java src/test/java/org/exist/xmlrpc/QuerySessionTest.java src/main/java/org/exist/xmlrpc/RpcAPI.java src/main/java/org/exist/xmlrpc/RpcConnection.java + src/main/java/org/exist/xmlrpc/XmldbRequestProcessorFactory.java src/main/java/org/exist/xmlrpc/XmlRpcExtensionConstants.java src/test/java/org/exist/xmlrpc/XmlRpcTest.java src/main/java/org/exist/xqj/Marshaller.java @@ -2148,6 +2172,7 @@ src/test/java/org/exist/xquery/CastExpressionTest.java src/test/java/org/exist/xquery/CleanupTest.java src/main/java/org/exist/xquery/CombiningExpression.java + src/main/java/org/exist/xquery/CompiledXQuery.java src/test/java/org/exist/xquery/ConstructedNodesRecoveryTest.java src/test/java/org/exist/xquery/ConstructedNodesTest.java src/main/java/org/exist/xquery/Context.java @@ -2199,6 +2224,9 @@ src/test/java/org/exist/xquery/PersistentDescendantOrSelfNodeKindTest.java src/test/java/org/exist/xquery/QueryPoolTest.java src/test/java/org/exist/xquery/RestBinariesTest.java + src/test/java/org/exist/xquery/RunningXQueryRestTest.java + src/test/java/org/exist/xquery/RunningXQueryTest.java + src/test/java/org/exist/xquery/RunningXQueryXmlDbTest.java src/test/java/org/exist/xquery/SeqOpTest.java src/test/java/org/exist/xquery/SpecialNamesTest.java src/test/java/org/exist/xquery/StoredModuleTest.java @@ -2387,6 +2415,7 @@ src/test/java/org/exist/xquery/functions/util/Base64FunctionsTest.java src/test/java/org/exist/xquery/functions/util/BaseConverterTest.java src/main/java/org/exist/xquery/functions/util/BuiltinFunctions.java + src/main/java/org/exist/xquery/functions/util/CollectionName.java src/main/java/org/exist/xquery/functions/util/DescribeFunction.java src/main/java/org/exist/xquery/functions/util/Eval.java src/test/java/org/exist/xquery/functions/util/EvalTest.java @@ -2665,6 +2694,13 @@ The original license statement is also included below.]]> src/main/java/org/exist/mediatype/MediaTypeService.java src/main/java/org/exist/mediatype/MediaTypeUtil.java + src/main/java/org/exist/security/internal/aider/ImmutableUnixStylePermissionAider.java + src/main/java/org/exist/source/DbStoreSource.java + src/main/java/org/exist/source/DbUriSource.java + src/main/java/org/exist/xmldb/LocalCompiledExpression.java + src/test/java/org/exist/xquery/RunningXQueryRestTest.java + src/test/java/org/exist/xquery/RunningXQueryTest.java + src/test/java/org/exist/xquery/RunningXQueryXmlDbTest.java src/main/resources/xyz/elemental/mediatype/media-type-aliases.xml src/main/resources/xyz/elemental/mediatype/media-type-mappings.xml src/main/resources/xyz/elemental/mediatype/mime.types @@ -2972,4 +3008,4 @@ The BaseX Team. The original license statement is also included below.]]> - \ No newline at end of file + diff --git a/exist-core/src/main/java/org/exist/collections/triggers/XQueryTrigger.java b/exist-core/src/main/java/org/exist/collections/triggers/XQueryTrigger.java index 534e8919b8..69cf17b3c8 100644 --- a/exist-core/src/main/java/org/exist/collections/triggers/XQueryTrigger.java +++ b/exist-core/src/main/java/org/exist/collections/triggers/XQueryTrigger.java @@ -61,7 +61,7 @@ import org.exist.dom.persistent.DocumentImpl; import org.exist.dom.QName; import org.exist.security.PermissionDeniedException; -import org.exist.source.DBSource; +import org.exist.source.DbStoreSource; import org.exist.source.Source; import org.exist.source.SourceFactory; import org.exist.source.StringSource; @@ -364,8 +364,8 @@ private CompiledXQuery getScript(final DBBroker broker, final Txn transaction) t context.prepareForReuse(); } - if (query instanceof DBSource) { - context.setModuleLoadPath(XmldbURI.EMBEDDED_SERVER_URI_PREFIX + ((DBSource)query).getDocumentPath().removeLastSegment().toString()); + if (query instanceof DbStoreSource) { + context.setModuleLoadPath(XmldbURI.EMBEDDED_SERVER_URI_PREFIX + ((DbStoreSource) query).getDocumentPath().removeLastSegment().toString()); } //compile the XQuery diff --git a/exist-core/src/main/java/org/exist/http/AuditTrailSessionListener.java b/exist-core/src/main/java/org/exist/http/AuditTrailSessionListener.java index 08d63fa7ef..4595a677f4 100644 --- a/exist-core/src/main/java/org/exist/http/AuditTrailSessionListener.java +++ b/exist-core/src/main/java/org/exist/http/AuditTrailSessionListener.java @@ -48,15 +48,13 @@ import com.evolvedbinary.j8fu.function.ConsumerE; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.exist.dom.persistent.BinaryDocument; -import org.exist.dom.persistent.LockedDocument; +import org.exist.EXistException; +import org.exist.security.PermissionDeniedException; import org.exist.security.Subject; -import org.exist.source.DBSource; +import org.exist.source.DbUriSource; import org.exist.source.Source; import org.exist.storage.BrokerPool; import org.exist.storage.DBBroker; -import org.exist.storage.XQueryPool; -import org.exist.storage.lock.Lock.LockMode; import org.exist.xmldb.XmldbURI; import org.exist.xquery.XPathException; import org.exist.xquery.XQueryContext; @@ -67,7 +65,7 @@ import javax.servlet.http.HttpSessionEvent; import javax.servlet.http.HttpSessionListener; -import javax.annotation.Nullable; +import java.io.IOException; import java.util.Optional; import java.util.Properties; @@ -116,50 +114,29 @@ private void executeXQuery(String xqueryResourcePath) { if (xqueryResourcePath != null && xqueryResourcePath.length() > 0) { xqueryResourcePath = xqueryResourcePath.trim(); - @Nullable Source source; - try { final BrokerPool pool = BrokerPool.getInstance(); - final XQueryPool xqpool = pool.getXQueryPool(); - final Subject sysSubject = pool.getSecurityManager().getSystemSubject(); - try (final DBBroker broker = pool.get(Optional.of(sysSubject))) { - if (broker == null) { - LOG.error("Unable to retrieve DBBroker for {}", sysSubject.getName()); - return; - } - - final XmldbURI pathUri = XmldbURI.create(xqueryResourcePath); + final XmldbURI pathUri = XmldbURI.create(xqueryResourcePath); + final Source source = DbUriSource.from(pool, sysSubject, pathUri, true, false); + + final ConsumerE setupXqueryContextPreCompilation = xqueryContext -> { + xqueryContext.setStaticallyKnownDocuments(new XmldbURI[]{pathUri}); + xqueryContext.setBaseURI(new AnyURIValue(pathUri.toString())); + }; - try (final LockedDocument lockedResource = broker.getXMLResource(pathUri, LockMode.READ_LOCK)) { + final Properties outputProperties = new Properties(); + try (final DBBroker broker = pool.get(Optional.of(sysSubject)); + final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, source, true, null, outputProperties, setupXqueryContextPreCompilation, null, null)) { - if (lockedResource != null) { - if (LOG.isTraceEnabled()) { - LOG.trace("Resource [{}] exists.", xqueryResourcePath); - } - source = new DBSource(broker, (BinaryDocument) lockedResource.getDocument(), true); - } else { - LOG.error("Resource [{}] does not exist.", xqueryResourcePath); - return; - } - - final ConsumerE setupXqueryContextPreCompilation = xqueryContext -> { - xqueryContext.setStaticallyKnownDocuments(new XmldbURI[]{pathUri}); - xqueryContext.setBaseURI(new AnyURIValue(pathUri.toString())); - }; - - final Properties outputProperties = new Properties(); - try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, source, true, null, outputProperties, setupXqueryContextPreCompilation, null, null)) { - - if (LOG.isTraceEnabled()) { - LOG.trace("XQuery execution results: {} in {}ms.", queryResult.result.toString(), queryResult.executionTime); - } - } + if (LOG.isTraceEnabled()) { + LOG.trace("XQuery execution results: {} in {}ms.", queryResult.result.toString(), queryResult.executionTime); } } - - } catch (final Exception e) { + } catch (final DbUriSource.NoSuchDocumentException e) { + LOG.error("Resource [{}] does not exist.", xqueryResourcePath); + } catch (final EXistException | PermissionDeniedException | XPathException | IOException e) { LOG.error("Exception while executing [{}] script", xqueryResourcePath, e); } } diff --git a/exist-core/src/main/java/org/exist/http/RESTServer.java b/exist-core/src/main/java/org/exist/http/RESTServer.java index 832026310b..964d280dd8 100644 --- a/exist-core/src/main/java/org/exist/http/RESTServer.java +++ b/exist-core/src/main/java/org/exist/http/RESTServer.java @@ -70,6 +70,7 @@ import org.exist.security.Subject; import org.exist.security.internal.RealmImpl; import org.exist.source.DBSource; +import org.exist.source.DbStoreSource; import org.exist.source.Source; import org.exist.source.StringSource; import org.exist.source.URLSource; @@ -578,7 +579,7 @@ public void doGet(final DBBroker broker, final Txn transaction, final HttpServle try { if (xquery_mime_type.equals(resource.getMediaType())) { // Execute the XQuery - executeXQuery(broker, transaction, resource, request, response, + executeXQuery(broker, transaction, lockedDocument, request, response, outputProperties, servletPath.toString(), pathInfo); } else if (xproc_mime_type.equals(resource.getMediaType())) { // Execute the XProc @@ -745,7 +746,7 @@ public void doPost(final DBBroker broker, final Txn transaction, final HttpServl try { if (xquery_mime_type.equals(resource.getMediaType())) { // Execute the XQuery - executeXQuery(broker, transaction, resource, request, response, + executeXQuery(broker, transaction, lockedDocument, request, response, outputProperties, servletPath.toString(), pathInfo); } else { // Execute the XProc @@ -1267,7 +1268,7 @@ private boolean checkForXQueryTarget(final DBBroker broker, final Txn transactio final Properties outputProperties = new Properties(defaultOutputKeysProperties); try { // Execute the XQuery - executeXQuery(broker, transaction, resource, request, response, + executeXQuery(broker, transaction, lockedDocument, request, response, outputProperties, servletPath.toString(), pathInfo); } catch (final XPathException e) { writeXPathExceptionHtml(response, HttpServletResponse.SC_BAD_REQUEST, UTF_8.name(), null, path.toString(), e); @@ -1386,10 +1387,12 @@ protected void search(final DBBroker broker, final Txn transaction, final String @Nullable final Item contextItem = extractContextItem(contextItemParam); final Sequence contextSequence = contextItem != null ? new ValueSequence(contextItem) : null; - try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, source, true, contextSequence, outputProperties, setupXqueryContextPreCompilation, setupXqueryContextPreExecution, setupXqueryContextPostExecution)) { + @Nullable XQueryUtil.QueryResult queryResult = null; + try { + queryResult = XQueryUtil.query(broker, source, true, contextSequence, outputProperties, setupXqueryContextPreCompilation, setupXqueryContextPreExecution, setupXqueryContextPostExecution); // special header to indicate that the query is not returned from cache - response.setHeader(XQUERY_CACHED_RESPONSE_HEADER, queryResult.compilationTime == XQueryUtil.QueryResult.RETRIEVED_CACHED_COMPILED_QUERY ? "true" : "false"); + response.setHeader(XQUERY_CACHED_RESPONSE_HEADER, queryResult.compilationTime == XQueryUtil.CompilationResult.RETRIEVED_CACHED_COMPILED_QUERY ? "true" : "false"); if (LOG.isDebugEnabled()) { LOG.debug("Found {} in {}ms.", queryResult.result.getItemCount(), queryResult.executionTime); @@ -1404,6 +1407,11 @@ protected void search(final DBBroker broker, final Txn transaction, final String } writeResults(response, broker, transaction, queryResult, howmany, start, typed, outputProperties, wrap); + } finally { + if (!cache && queryResult != null) { + // NOTE(AR) we can only close the query result if we are not caching it for reuse in the sessionManager, otherwise it has to be closed when it is later removed from the sessionManager + queryResult.close(); + } } } catch (final IOException e) { @@ -1575,41 +1583,64 @@ private void declareExternalAndXQJVariables(final XQueryContext context, /** * Directly execute an XQuery stored as a binary document in the database. * - * @throws PermissionDeniedException + * NOTE The document will be unlocked after query compilation (only if compilation succeeds). + * + * @param broker the database broker. + * @param transaction the database transaction. + * @param lockedQueryDocument the locked document holding the query to be executed. + * @param request the HTTP request. + * @param response the HTTP response. + * @param outputProperties any serialization properties. + * @param servletPath the path to the servlet. + * @param pathInfo the path. + * + * @throws XPathException if an error occurs during query compilation or execution. + * @throws BadRequestException if the request had a problem. + * @throws PermissionDeniedException if the calling user does not have sufficient permissions. */ - private void executeXQuery(final DBBroker broker, final Txn transaction, final DocumentImpl resource, + private void executeXQuery(final DBBroker broker, final Txn transaction, final LockedDocument lockedQueryDocument, final HttpServletRequest request, final HttpServletResponse response, final Properties outputProperties, final String servletPath, final String pathInfo) throws XPathException, BadRequestException, PermissionDeniedException { - final Source source = new DBSource(broker, (BinaryDocument) resource, true); + final DocumentImpl resource = lockedQueryDocument.getDocument(); + final DbStoreSource source = new DBSource(broker.getBrokerPool(), (BinaryDocument) resource, true); final ConsumerE setupXqueryContextPreCompilation = xqueryContext -> { xqueryContext.setModuleLoadPath(XmldbURI.EMBEDDED_SERVER_URI.append(resource.getCollection().getURI()).toString()); xqueryContext.setStaticallyKnownDocuments(new XmldbURI[]{ resource.getCollection().getURI() }); }; - final ConsumerE setupXqueryContextPreExecution = xqueryContext -> { - final HttpRequestWrapper reqw = declareVariables(xqueryContext, null, request, response); - reqw.setServletPath(servletPath); - reqw.setPathInfo(pathInfo); - DebuggeeFactory.checkForDebugRequest(request, xqueryContext); - }; + try { + // NOTE(AR) compilationResult will be cleaned up by the try-with-resources wrapped XQueryUtil.execute(...); at present between here and there no exceptions can occur + final XQueryUtil.CompilationResult compilationResult = XQueryUtil.compile(broker, source, false, true, setupXqueryContextPreCompilation); - final BiConsumerE setupXqueryContextPostExecution = (xqueryContext, queryResult) -> { - // Pass last modified date to the HTTP response - HTTPUtils.addLastModifiedHeader(queryResult.result, xqueryContext); - }; + // NOTE(AR) query is now compiled so we can release the LockedDocument eagerly + lockedQueryDocument.close(); - try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, source, true, null, outputProperties, setupXqueryContextPreCompilation, setupXqueryContextPreExecution, setupXqueryContextPostExecution)) { - // Special header to indicate whether the compiled query is returned from the cache - response.setHeader(XQUERY_CACHED_RESPONSE_HEADER, queryResult.compilationTime == XQueryUtil.QueryResult.RETRIEVED_CACHED_COMPILED_QUERY ? "true" : "false"); + final ConsumerE setupXqueryContextPreExecution = xqueryContext -> { + final HttpRequestWrapper reqw = declareVariables(xqueryContext, null, request, response); + reqw.setServletPath(servletPath); + reqw.setPathInfo(pathInfo); + DebuggeeFactory.checkForDebugRequest(request, xqueryContext); + }; + + final BiConsumerE setupXqueryContextPostExecution = (xqueryContext, queryResult) -> { + // Pass last modified date to the HTTP response + HTTPUtils.addLastModifiedHeader(queryResult.result, xqueryContext); + }; + + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.execute(broker, compilationResult, null, outputProperties, setupXqueryContextPreExecution, setupXqueryContextPostExecution)) { + // Special header to indicate whether the compiled query is returned from the cache + response.setHeader(XQUERY_CACHED_RESPONSE_HEADER, queryResult.compilationTime == XQueryUtil.CompilationResult.RETRIEVED_CACHED_COMPILED_QUERY ? "true" : "false"); + + final boolean wrap = "yes".equals(outputProperties.getProperty("_wrap")); + writeResults(response, broker, transaction, queryResult, -1, 1, false, outputProperties, wrap); + } - final boolean wrap = "yes".equals(outputProperties.getProperty("_wrap")); - writeResults(response, broker, transaction, queryResult, -1, 1, false, outputProperties, wrap); } catch (final IOException e) { - throw new BadRequestException("Failed to read query from " + resource.getURI(), e); + throw new BadRequestException("Failed to read query from: " + source.getDocumentPath(), e); } } @@ -1663,7 +1694,7 @@ private void executeXProc(final DBBroker broker, final Txn transaction, final Do try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, source, true, null, null, setupXqueryContextPreCompilation, setupXqueryContextPreExecution, setupXqueryContextPostExecution)) { // special header to indicate that the query is not returned from cache - response.setHeader(XQUERY_CACHED_RESPONSE_HEADER, queryResult.compilationTime == XQueryUtil.QueryResult.RETRIEVED_CACHED_COMPILED_QUERY ? "true" : "false"); + response.setHeader(XQUERY_CACHED_RESPONSE_HEADER, queryResult.compilationTime == XQueryUtil.CompilationResult.RETRIEVED_CACHED_COMPILED_QUERY ? "true" : "false"); writeResults(response, broker, transaction, queryResult, -1, 1, false, outputProperties, false); } catch (final IOException e) { diff --git a/exist-core/src/main/java/org/exist/http/SessionManager.java b/exist-core/src/main/java/org/exist/http/SessionManager.java index 9372250c4e..b4ead5748c 100644 --- a/exist-core/src/main/java/org/exist/http/SessionManager.java +++ b/exist-core/src/main/java/org/exist/http/SessionManager.java @@ -47,11 +47,13 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.RemovalListener; import net.jcip.annotations.ThreadSafe; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.exist.xquery.XQueryUtil; +import javax.annotation.Nullable; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -64,6 +66,15 @@ public class SessionManager { private static final Logger LOG = LogManager.getLogger(SessionManager.class); private static final long TIMEOUT = 120_000; // ms (e.g. 2 minutes) + private static final RemovalListener REMOVAL_LISTENER = (sessionId, queryAndResult, removalCause) -> { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing cached query result for session: {}", sessionId); + } + + // NOTE(AR) make sure to release any resources still held by the query result and potentially send it back to the query pool for reuse + queryAndResult.result.close(); + }; + private final AtomicInteger sessionIdCounter = new AtomicInteger(); private final Cache cache; @@ -78,12 +89,10 @@ private QueryAndResult(final String query, final XQueryUtil.QueryResult result) } public SessionManager() { - final Caffeine cacheBuilder = Caffeine.newBuilder() - .expireAfterAccess(TIMEOUT, TimeUnit.MILLISECONDS); - if(LOG.isDebugEnabled()) { - cacheBuilder.removalListener((key, value, cause) -> LOG.debug("Removing cached query result for session: {}", key)); - } - cache = cacheBuilder.build(); + this.cache = Caffeine.newBuilder() + .expireAfterAccess(TIMEOUT, TimeUnit.MILLISECONDS) + .removalListener(REMOVAL_LISTENER) + .build(); } public int add(final String query, final XQueryUtil.QueryResult result) { @@ -92,7 +101,7 @@ public int add(final String query, final XQueryUtil.QueryResult result) { return sessionId; } - public XQueryUtil.QueryResult get(final String query, final int sessionId) { + public @Nullable XQueryUtil.QueryResult get(final String query, final int sessionId) { if (sessionId < 0 || sessionId >= sessionIdCounter.get()) { return null; // out of scope } diff --git a/exist-core/src/main/java/org/exist/http/servlets/RedirectorServlet.java b/exist-core/src/main/java/org/exist/http/servlets/RedirectorServlet.java index 5583554ee6..dab483c2b6 100644 --- a/exist-core/src/main/java/org/exist/http/servlets/RedirectorServlet.java +++ b/exist-core/src/main/java/org/exist/http/servlets/RedirectorServlet.java @@ -302,7 +302,7 @@ private XQueryUtil.QueryResult executeQuery(final Source source, final RequestWr final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, source, true, null, outputProperties, setupXqueryContextPreCompilation, null, null); // special header to indicate that the query is not returned from cache - response.setHeader(XQUERY_CACHED_RESPONSE_HEADER, queryResult.compilationTime == XQueryUtil.QueryResult.RETRIEVED_CACHED_COMPILED_QUERY ? "true" : "false"); + response.setHeader(XQUERY_CACHED_RESPONSE_HEADER, queryResult.compilationTime == XQueryUtil.CompilationResult.RETRIEVED_CACHED_COMPILED_QUERY ? "true" : "false"); return queryResult; } diff --git a/exist-core/src/main/java/org/exist/http/servlets/XQueryServlet.java b/exist-core/src/main/java/org/exist/http/servlets/XQueryServlet.java index 004a9eb625..ed543a22ed 100644 --- a/exist-core/src/main/java/org/exist/http/servlets/XQueryServlet.java +++ b/exist-core/src/main/java/org/exist/http/servlets/XQueryServlet.java @@ -421,17 +421,13 @@ protected void process(HttpServletRequest request, HttpServletResponse response) // System.out.println("path="+path); if(descriptor.allowSource(path)) { - if (source instanceof DBSource) { - try { - ((DBSource) source).validate(user, Permission.READ); - } catch (final PermissionDeniedException e) { - if (getDefaultUser().equals(user)) { - getAuthenticator().sendChallenge(request, response); - } else { - response.sendError(HttpServletResponse.SC_FORBIDDEN, "Permission to view XQuery source for: " + path + " denied. (no read access)"); - } - return; + if (!source.getPermissions().validate(user, Permission.READ)) { + if (getDefaultUser().equals(user)) { + getAuthenticator().sendChallenge(request, response); + } else { + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Permission to view XQuery source for: " + path + " denied. (no read access)"); } + return; } //Show the source of the XQuery @@ -512,7 +508,7 @@ protected void process(HttpServletRequest request, HttpServletResponse response) try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, source, true, null, outputProperties, null, setupXqueryContextPreExecution, setupXqueryContextPostExecution)) { // special header to indicate that the query is not returned from cache - response.setHeader(XQUERY_CACHED_RESPONSE_HEADER, queryResult.compilationTime == XQueryUtil.QueryResult.RETRIEVED_CACHED_COMPILED_QUERY ? "true" : "false"); + response.setHeader(XQUERY_CACHED_RESPONSE_HEADER, queryResult.compilationTime == XQueryUtil.CompilationResult.RETRIEVED_CACHED_COMPILED_QUERY ? "true" : "false"); final String mediaType = outputProperties.getProperty(OutputKeys.MEDIA_TYPE); if (mediaType != null) { diff --git a/exist-core/src/main/java/org/exist/http/urlrewrite/XQueryURLRewrite.java b/exist-core/src/main/java/org/exist/http/urlrewrite/XQueryURLRewrite.java index b27aa32de5..38385b4ae8 100644 --- a/exist-core/src/main/java/org/exist/http/urlrewrite/XQueryURLRewrite.java +++ b/exist-core/src/main/java/org/exist/http/urlrewrite/XQueryURLRewrite.java @@ -53,7 +53,6 @@ import org.exist.EXistException; import org.exist.Namespaces; import org.exist.collections.Collection; -import org.exist.dom.persistent.BinaryDocument; import org.exist.dom.persistent.DocumentImpl; import org.exist.dom.persistent.LockedDocument; import org.exist.http.Descriptor; @@ -66,7 +65,7 @@ import org.exist.security.PermissionDeniedException; import org.exist.security.Subject; import org.exist.security.internal.web.HttpAccount; -import org.exist.source.DBSource; +import org.exist.source.DbUriSource; import org.exist.source.FileSource; import org.exist.source.Source; import org.exist.source.SourceFactory; @@ -504,11 +503,15 @@ private ModelAndView getFromCache(final String url, final Subject user) throws E try (final DBBroker broker = pool.get(Optional.ofNullable(user))) { - if (model.getSourceInfo().source instanceof DBSource) { - ((DBSource) model.getSourceInfo().source).validate(Permission.EXECUTE); + try { + if (!model.getSourceInfo().source.getPermissions().validate(broker.getCurrentSubject(), Permission.EXECUTE)) { + throw new PermissionDeniedException("Subject '" + broker.getCurrentSubject().getName() + "' does not have read access to resource '" + model.sourceInfo.source.pathOrShortIdentifier() + "'."); + } + } catch (final IOException e) { + throw new PermissionDeniedException("Subject '" + broker.getCurrentSubject().getName() + "' does not have read access to resource '" + model.sourceInfo.source.pathOrShortIdentifier() + "'.", e); } - if (model.getSourceInfo().source.isValid(broker) != Source.Validity.VALID) { + if (model.getSourceInfo().source.isValid() != Source.Validity.VALID) { urlCache.remove(url); return null; } @@ -770,7 +773,8 @@ SourceInfo findSourceFromDb(final DBBroker broker, final String basePath, final } final String controllerPath = controllerDoc.getCollection().getURI().getRawCollectionPath(); - return new SourceInfo(new DBSource(broker, (BinaryDocument) controllerDoc, true), "xmldb:exist://" + controllerPath, controllerPath.substring(locationUri.getCollectionPath().length())); + final Source source = DbUriSource.from(broker.getBrokerPool(), broker.getCurrentSubject(), controllerDoc, true, false); + return new SourceInfo(source, "xmldb:exist://" + controllerPath, controllerPath.substring(locationUri.getCollectionPath().length())); } catch (final URISyntaxException e) { LOG.warn("Bad URI for base path: {}", e.getMessage(), e); @@ -918,8 +922,8 @@ private SourceInfo getSource(final DBBroker broker, final String moduleLoadPath) throw new ServletException("XQuery resource: " + query + " is not an XQuery or " + "declares a wrong mime-type"); } - sourceInfo = new SourceInfo(new DBSource(broker, (BinaryDocument) sourceDoc, true), - locationUri.toString()); + final Source source = DbUriSource.from(broker.getBrokerPool(), broker.getCurrentSubject(), sourceDoc, true, false); + sourceInfo = new SourceInfo(source, locationUri.toString()); } catch (final PermissionDeniedException e) { throw new ServletException("permission denied to read module source from " + query); } diff --git a/exist-core/src/main/java/org/exist/jetty/JettyStart.java b/exist-core/src/main/java/org/exist/jetty/JettyStart.java index a524dd8470..0a2604499a 100644 --- a/exist-core/src/main/java/org/exist/jetty/JettyStart.java +++ b/exist-core/src/main/java/org/exist/jetty/JettyStart.java @@ -59,10 +59,13 @@ import org.eclipse.jetty.xml.XmlConfiguration; import org.exist.SystemProperties; import org.exist.http.servlets.ExistExtensionServlet; +import org.exist.repo.AutoDeploymentTrigger; import org.exist.start.CompatibleJavaVersionCheck; import org.exist.start.Main; import org.exist.start.StartException; import org.exist.storage.BrokerPool; +import org.exist.storage.BrokerPoolConstants; +import org.exist.util.Configuration; import org.exist.util.ConfigurationHelper; import org.exist.util.FileUtils; import org.exist.util.OSUtil; @@ -78,6 +81,7 @@ import se.softhouse.jargo.CommandLineParser; import se.softhouse.jargo.ParsedArguments; +import javax.annotation.Nullable; import java.io.IOException; import java.io.InputStream; import java.io.LineNumberReader; @@ -133,7 +137,8 @@ public class JettyStart extends Observable implements LifeCycle.Listener { @GuardedBy("this") private int status = STATUS_STOPPED; @GuardedBy("this") private Optional shutdownHookThread = Optional.empty(); @GuardedBy("this") private int primaryPort = 8080; - + private final Properties additionalElementalConfigProperties; + private final Properties additionalJettyConfigProperties; public static void main(final String[] args) { try { @@ -165,7 +170,17 @@ public static void main(final String[] args) { } public JettyStart() { - // Additional checks XML libs @@@@ + this(null, null); + } + + /** + * @param additionalElementalConfigProperties any additional Elemental configuration properties. + * @param additionalJettyConfigProperties any additional Jetty configuration properties + */ + public JettyStart(@Nullable final Properties additionalElementalConfigProperties, @Nullable final Properties additionalJettyConfigProperties) { + this.additionalElementalConfigProperties = additionalElementalConfigProperties != null ? additionalElementalConfigProperties : new Properties(); + this.additionalJettyConfigProperties = additionalJettyConfigProperties != null ? additionalJettyConfigProperties : new Properties(); + // Additional checks XML libs XmlLibraryChecker.check(); } @@ -174,11 +189,11 @@ public synchronized void run() { } public synchronized void run(final boolean standalone) { - final String jettyProperty = Optional.ofNullable(System.getProperty(JETTY_HOME_PROP)) + final String jettyHome = Optional.ofNullable(System.getProperty(JETTY_HOME_PROP)) .orElseGet(() -> { final Optional home = ConfigurationHelper.getExistHome(); - final Path jettyHome = FileUtils.resolve(home, "tools").resolve("jetty"); - final String jettyPath = jettyHome.toAbsolutePath().toString(); + final Path toolsJetty = FileUtils.resolve(home, "tools").resolve("jetty"); + final String jettyPath = toolsJetty.toAbsolutePath().toString(); System.setProperty(JETTY_HOME_PROP, jettyPath); return jettyPath; }); @@ -187,9 +202,9 @@ public synchronized void run(final boolean standalone) { final Path jettyConfig; if (standalone) { - jettyConfig = Paths.get(jettyProperty).normalize().resolve("etc").resolve(Main.STANDALONE_ENABLED_JETTY_CONFIGS); + jettyConfig = Paths.get(jettyHome).normalize().resolve("etc").resolve(Main.STANDALONE_ENABLED_JETTY_CONFIGS); } else { - jettyConfig = Paths.get(jettyProperty).normalize().resolve("etc").resolve(Main.STANDARD_ENABLED_JETTY_CONFIGS); + jettyConfig = Paths.get(jettyHome).normalize().resolve("etc").resolve(Main.STANDARD_ENABLED_JETTY_CONFIGS); } run(new String[] { jettyConfig.toAbsolutePath().toString() }, null); } @@ -223,16 +238,21 @@ public synchronized void run(final String[] args, final Observer observer) { } } - final Map configProperties; + final Map jettyConfigProperties; try { - configProperties = getConfigProperties(jettyConfig.getParent()); + jettyConfigProperties = getJettyConfigProperties(jettyConfig.getParent()); // modify JETTY_HOME and JETTY_BASE properties when running with classpath config if (configFromClasspath) { final String jettyClasspathHome = jettyConfig.getParent().getParent().toAbsolutePath().toString(); System.setProperty(JETTY_HOME_PROP, jettyClasspathHome); - configProperties.put(JETTY_HOME_PROP, jettyClasspathHome); - configProperties.put(JETTY_BASE_PROP, jettyClasspathHome); + jettyConfigProperties.put(JETTY_HOME_PROP, jettyClasspathHome); + jettyConfigProperties.put(JETTY_BASE_PROP, jettyClasspathHome); + } + + // override any specified Jetty config properties + for (final Map.Entry additionalJettyConfigProperty : additionalJettyConfigProperties.entrySet()) { + jettyConfigProperties.put(additionalJettyConfigProperty.getKey().toString(), additionalJettyConfigProperty.getValue().toString()); } if (observer != null) { @@ -271,10 +291,29 @@ public synchronized void run(final String[] args, final Observer observer) { logger.info("[Log4j Configuration: {}]", System.getProperty("log4j.configurationFile")); logger.info("[Jetty Version: {}]", Jetty.VERSION); - logger.info("[Jetty Home: {}]", configProperties.get(JETTY_HOME_PROP)); - logger.info("[Jetty Base: {}]", configProperties.get(JETTY_BASE_PROP)); + logger.info("[Jetty Home: {}]", jettyConfigProperties.get(JETTY_HOME_PROP)); + logger.info("[Jetty Base: {}]", jettyConfigProperties.get(JETTY_BASE_PROP)); logger.info("[Jetty Configuration: {}]", jettyConfig.toAbsolutePath().toString()); + // override any specified Elemental config properties + for (final Map.Entry additionalElementalConfigProperty : additionalElementalConfigProperties.entrySet()) { + final Object additionalElementalConfigPropertyKey = additionalElementalConfigProperty.getKey(); + final Object additionalElementalConfigPropertyValue = additionalElementalConfigProperty.getValue(); + if (AUTODEPLOY_PROPERTY.equals(additionalElementalConfigPropertyKey) && "off".equals(additionalElementalConfigPropertyValue)) { + // remove auto deploy from config if present + final List configuredStartupTriggers = (List) config.getProperty(BrokerPoolConstants.PROPERTY_STARTUP_TRIGGERS); + for (final Configuration.StartupTriggerConfig configuredStartupTrigger : configuredStartupTriggers) { + if (AutoDeploymentTrigger.class.getName().equals(configuredStartupTrigger.getClazz())) { + configuredStartupTriggers.remove(configuredStartupTrigger); + break; + } + } + + } else { + config.setProperty(additionalElementalConfigPropertyKey.toString(), additionalElementalConfigPropertyValue); + } + } + BrokerPool.configure(1, 5, config, Optional.ofNullable(observer)); // register the XMLDB driver @@ -290,21 +329,21 @@ public synchronized void run(final String[] args, final Observer observer) { try { // load jetty configurations - final List configFiles = getEnabledConfigFiles(jettyConfig); + final List jettyConfigFiles = getEnabledJettyConfigFiles(jettyConfig); final List configuredObjects = new ArrayList<>(); - XmlConfiguration last = null; - for(final Path confFile : configFiles) { + XmlConfiguration lastJettyConfiguration = null; + for(final Path jettyConfigFile : jettyConfigFiles) { if (logger.isDebugEnabled()) { - logger.debug("[Loading Jetty Configuration: {}]", confFile.toString()); + logger.debug("[Loading Jetty Configuration: {}]", jettyConfigFile.toString()); } - try(final InputStream is = Files.newInputStream(confFile)) { - final XmlConfiguration configuration = new XmlConfiguration(is); - if (last != null) { - configuration.getIdMap().putAll(last.getIdMap()); + try(final InputStream is = Files.newInputStream(jettyConfigFile)) { + final XmlConfiguration jettyConfiguration = new XmlConfiguration(is); + if (lastJettyConfiguration != null) { + jettyConfiguration.getIdMap().putAll(lastJettyConfiguration.getIdMap()); } - configuration.getProperties().putAll(configProperties); - configuredObjects.add(configuration.configure()); - last = configuration; + jettyConfiguration.getProperties().putAll(jettyConfigProperties); + configuredObjects.add(jettyConfiguration.configure()); + lastJettyConfiguration = jettyConfiguration; } } @@ -551,7 +590,7 @@ private Optional startJetty(final List configuredObjects) throws return server; } - private Map getConfigProperties(final Path configDir) throws IOException { + private Map getJettyConfigProperties(final Path configDir) throws IOException { final Map configProperties = new HashMap<>(); //load jetty.properties file @@ -575,7 +614,7 @@ private Map getConfigProperties(final Path configDir) throws IOE return configProperties; } - private List getEnabledConfigFiles(final Path enabledJettyConfigs) throws IOException { + private List getEnabledJettyConfigFiles(final Path enabledJettyConfigs) throws IOException { if(Files.notExists(enabledJettyConfigs)) { throw new IOException("Cannot find config enabler: " + enabledJettyConfigs.toString()); } else { diff --git a/exist-core/src/main/java/org/exist/repo/ExistRepository.java b/exist-core/src/main/java/org/exist/repo/ExistRepository.java index d40811102e..e1bf8b6642 100644 --- a/exist-core/src/main/java/org/exist/repo/ExistRepository.java +++ b/exist-core/src/main/java/org/exist/repo/ExistRepository.java @@ -48,13 +48,16 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.exist.dom.persistent.BinaryDocument; +import org.exist.dom.persistent.DocumentImpl; +import org.exist.dom.persistent.LockedDocument; import org.exist.security.PermissionDeniedException; -import org.exist.source.DBSource; +import org.exist.source.DbUriSource; import org.exist.storage.BrokerPool; import org.exist.storage.BrokerPoolService; import org.exist.storage.BrokerPoolServiceException; import org.exist.storage.DBBroker; import org.exist.storage.NativeBroker; +import org.exist.storage.lock.Lock; import org.exist.util.Configuration; import org.exist.util.FileUtils; import org.exist.xmldb.XmldbURI; @@ -70,7 +73,6 @@ import org.expath.pkg.repo.PackageException; import org.expath.pkg.repo.Repository; import org.expath.pkg.repo.URISpace; -import org.w3c.dom.Document; import javax.annotation.Nullable; import javax.xml.transform.Source; @@ -323,16 +325,24 @@ public Path resolveXQueryModule(final String namespace) throws XPathException { // 1. attempt to locate it within a library XmldbURI xqueryDbPath = XmldbURI.create("xmldb:exist:///db/system/repo/" + relXQueryPath); - @Nullable Document doc = broker.getXMLResource(xqueryDbPath); - if (doc != null && doc instanceof BinaryDocument) { - return new DBSource(broker, (BinaryDocument) doc, false); + try (@Nullable final LockedDocument lockedDoc = broker.getXMLResource(xqueryDbPath, Lock.LockMode.READ_LOCK)) { + if (lockedDoc != null) { + final DocumentImpl doc = lockedDoc.getDocument(); + if (doc instanceof BinaryDocument) { + return DbUriSource.from(broker.getBrokerPool(), broker.getCurrentSubject(), doc, false, false); + } + } } // 2. attempt to locate it within an app xqueryDbPath = XmldbURI.create("xmldb:exist:///db/apps/" + relXQueryPath); - doc = broker.getXMLResource(xqueryDbPath); - if (doc != null && doc instanceof BinaryDocument) { - return new DBSource(broker, (BinaryDocument) doc, false); + try (@Nullable LockedDocument lockedDoc = broker.getXMLResource(xqueryDbPath, Lock.LockMode.READ_LOCK)) { + if (lockedDoc != null) { + final DocumentImpl doc = lockedDoc.getDocument(); + if (doc instanceof BinaryDocument) { + return DbUriSource.from(broker.getBrokerPool(), broker.getCurrentSubject(), doc, false, false); + } + } } return null; diff --git a/exist-core/src/main/java/org/exist/scheduler/UserXQueryJob.java b/exist-core/src/main/java/org/exist/scheduler/UserXQueryJob.java index 09fb4327a3..4c5db487b4 100644 --- a/exist-core/src/main/java/org/exist/scheduler/UserXQueryJob.java +++ b/exist-core/src/main/java/org/exist/scheduler/UserXQueryJob.java @@ -54,16 +54,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.exist.EXistException; -import org.exist.dom.persistent.BinaryDocument; -import org.exist.dom.persistent.LockedDocument; import org.exist.security.PermissionDeniedException; import org.exist.security.Subject; -import org.exist.source.DBSource; +import org.exist.source.DbStoreSource; +import org.exist.source.DbUriSource; import org.exist.source.Source; import org.exist.source.SourceFactory; import org.exist.storage.BrokerPool; import org.exist.storage.DBBroker; -import org.exist.storage.lock.Lock.LockMode; import org.exist.xmldb.XmldbURI; import org.exist.xquery.XPathException; import org.exist.xquery.XQueryContext; @@ -175,24 +173,22 @@ public final void execute(final JobExecutionContext jec) throws JobExecutionExce } try (final DBBroker broker = pool.get(Optional.of(user))) { - if(xqueryResource.indexOf(':') > 0) { + if (xqueryResource.indexOf(':') > 0) { final Source source = SourceFactory.getSource(broker, "", xqueryResource, true); - if(source != null) { - executeXQuery(pool, broker, source, params); + if (source != null) { + executeXQuery(pool, broker, source, params); return; } } else { final XmldbURI pathUri = XmldbURI.create(xqueryResource); - try(final LockedDocument lockedResource = broker.getXMLResource(pathUri, LockMode.READ_LOCK)) { - if (lockedResource != null) { - final Source source = new DBSource(broker, (BinaryDocument) lockedResource.getDocument(), true); - executeXQuery(pool, broker, source, params); - return; - } - } + final Source source = DbUriSource.from(pool, pathUri, true, false); + executeXQuery(pool, broker, source, params); + return; } LOG.warn("XQuery User Job not found: {}, job not scheduled", xqueryResource); + } catch (final DbUriSource.NoSuchDocumentException e) { + abort("Could not load XQuery: " + e.getMessage()); } catch(final EXistException ee) { abort("Could not get DBBroker!"); } catch(final PermissionDeniedException pde) { @@ -207,8 +203,8 @@ public final void execute(final JobExecutionContext jec) throws JobExecutionExce private void executeXQuery(final BrokerPool pool, final DBBroker broker, final Source source, final Properties params) throws PermissionDeniedException, XPathException, JobExecutionException { final ConsumerE setupXqueryContextPreCompilation = xqueryContext -> { - if (source instanceof DBSource) { - final XmldbURI collectionUri = ((DBSource) source).getDocumentPath().removeLastSegment(); + if (source instanceof DbStoreSource) { + final XmldbURI collectionUri = ((DbStoreSource) source).getDocumentPath().removeLastSegment(); xqueryContext.setModuleLoadPath(XmldbURI.EMBEDDED_SERVER_URI.append(collectionUri.getCollectionPath()).toString()); xqueryContext.setStaticallyKnownDocuments(new XmldbURI[]{collectionUri}); } @@ -253,4 +249,4 @@ private void abort(final String message, final boolean unschedule) throws JobExe throw jaa; } -} \ No newline at end of file +} diff --git a/exist-core/src/main/java/org/exist/security/Permission.java b/exist-core/src/main/java/org/exist/security/Permission.java index 5dd869743d..51d4e75b60 100644 --- a/exist-core/src/main/java/org/exist/security/Permission.java +++ b/exist-core/src/main/java/org/exist/security/Permission.java @@ -258,11 +258,11 @@ public interface Permission { void setSticky(boolean sticky) throws PermissionDeniedException; /** - * Check if user has the requested mode for this resource. + * Check if user has the requested mode for this resource. * - *@param user The user - *@param mode The requested mode - *@return true if user has the requested mode + * @param user The user + * @param mode The requested mode + * @return true if user has the requested mode */ boolean validate(Subject user, int mode); diff --git a/exist-core/src/main/java/org/exist/security/UnixStylePermissionInternal.java b/exist-core/src/main/java/org/exist/security/UnixStylePermissionInternal.java index 604d388b0d..4546a85ca0 100644 --- a/exist-core/src/main/java/org/exist/security/UnixStylePermissionInternal.java +++ b/exist-core/src/main/java/org/exist/security/UnixStylePermissionInternal.java @@ -429,13 +429,6 @@ public String toString() { return String.valueOf(ch); } - /** - * Check if user has the requested mode for this resource. - * - * @param user The user - * @param mode The requested mode - * @return true if user has the requested mode - */ @Override public boolean validate(final Subject user, final int mode) { diff --git a/exist-core/src/main/java/org/exist/security/internal/SMEvents.java b/exist-core/src/main/java/org/exist/security/internal/SMEvents.java index ea410da8ea..dbe8ff6571 100644 --- a/exist-core/src/main/java/org/exist/security/internal/SMEvents.java +++ b/exist-core/src/main/java/org/exist/security/internal/SMEvents.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -24,30 +48,32 @@ import java.util.List; import java.util.Optional; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.exist.Database; +import org.exist.EXistException; import org.exist.config.Configurable; import org.exist.config.Configuration; import org.exist.config.Configurator; import org.exist.config.annotation.ConfigurationClass; import org.exist.config.annotation.ConfigurationFieldAsAttribute; import org.exist.config.annotation.ConfigurationFieldAsElement; -import org.exist.dom.persistent.BinaryDocument; -import org.exist.dom.persistent.LockedDocument; import org.exist.dom.persistent.NodeSet; import org.exist.dom.QName; import org.exist.security.PermissionDeniedException; import org.exist.security.SecurityManager; import org.exist.security.Subject; -import org.exist.source.DBSource; +import org.exist.source.DbUriSource; import org.exist.source.Source; import org.exist.source.StringSource; import org.exist.storage.DBBroker; import org.exist.storage.ProcessMonitor; -import org.exist.storage.lock.Lock.LockMode; import org.exist.xmldb.XmldbURI; import org.exist.xquery.*; import org.exist.xquery.value.Sequence; +import javax.annotation.Nullable; + /** * @author Dmitriy Shabanov * @@ -58,6 +84,8 @@ public class SMEvents implements Configurable { public final static String NAMESPACE_URI = "http://exist-db.org/security/events"; public final static String PREFIX = "sec-ev"; //security-events //secev //sev + private static final Logger LOG = LogManager.getLogger(SMEvents.class); + @ConfigurationFieldAsAttribute("script-uri") protected String scriptURI = ""; @@ -134,8 +162,7 @@ protected void runScript(Subject subject, String scriptURI, String script, QName call.eval(contextSequence, null); } } catch(final XPathException e) { - //XXX: log - e.printStackTrace(); + LOG.error(e.getMessage(), e); } finally { if (pm != null) { context.getProfiler().traceQueryEnd(context); @@ -147,30 +174,22 @@ protected void runScript(Subject subject, String scriptURI, String script, QName } } catch (final Exception e) { - //XXX: log - e.printStackTrace(); + LOG.error(e.getMessage(), e); } } - private Source getQuerySource(DBBroker broker, String scriptURI, String script) { - if(scriptURI != null) { - + private @Nullable Source getQuerySource(final DBBroker broker, final String scriptURI, final String script) { + if (scriptURI != null) { final XmldbURI pathUri = XmldbURI.create(scriptURI); - try(final LockedDocument lockedResource = broker.getXMLResource(pathUri, LockMode.READ_LOCK)) { - if (lockedResource != null) { - return new DBSource(broker, (BinaryDocument)lockedResource.getDocument(), true); - } - } catch (final PermissionDeniedException e) { - //XXX: log - e.printStackTrace(); + try { + return DbUriSource.from(broker.getBrokerPool(), broker.getCurrentSubject(), pathUri, true, false); + } catch (final DbUriSource.NoSuchDocumentException e) { + return null; + } catch (final EXistException | PermissionDeniedException e) { + LOG.error(e.getMessage(), e); } -// try { -// querySource = SourceFactory.getSource(broker, null, scriptURI, false); -// } catch(Exception e) { -// //LOG.error(e); -// } - } else if(script != null && !script.isEmpty()) { + } else if (script != null && !script.isEmpty()) { return new StringSource(script); } @@ -186,4 +205,4 @@ public boolean isConfigured() { public Configuration getConfiguration() { return configuration; } -} \ No newline at end of file +} diff --git a/exist-core/src/main/java/org/exist/security/internal/aider/ImmutableUnixStylePermissionAider.java b/exist-core/src/main/java/org/exist/security/internal/aider/ImmutableUnixStylePermissionAider.java new file mode 100644 index 0000000000..7e2bc902cb --- /dev/null +++ b/exist-core/src/main/java/org/exist/security/internal/aider/ImmutableUnixStylePermissionAider.java @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2014, Evolved Binary Ltd + * + * This file was originally ported from FusionDB to Elemental by + * Evolved Binary, for the benefit of the Elemental Open Source community. + * Only the ported code as it appears in this file, at the time that + * it was contributed to Elemental, was re-licensed under The GNU + * Lesser General Public License v2.1 only for use in Elemental. + * + * This license grant applies only to a snapshot of the code as it + * appeared when ported, it does not offer or infer any rights to either + * updates of this source code or access to the original source code. + * + * The GNU Lesser General Public License v2.1 only license follows. + * + * ===================================================================== + * + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.security.internal.aider; + +import org.exist.security.Account; +import org.exist.security.Group; +import org.exist.security.Permission; +import org.exist.storage.io.VariableByteInput; + +import java.io.IOException; + +/** + * An Immutable version of {@link UnixStylePermissionAider}. + */ +public class ImmutableUnixStylePermissionAider extends UnixStylePermissionAider { + + /** + * Construct a Permission with given mode + * + */ + public ImmutableUnixStylePermissionAider() { + super(); + } + + /** + * Construct a Permission with given mode + * + * @param mode The mode + */ + public ImmutableUnixStylePermissionAider(final int mode) { + super(mode); + } + + + /** + * Construct a permission with given user, group and mode + * + * @param user name of the owner + * @param group name of the group + * @param mode mode for the resource. + */ + public ImmutableUnixStylePermissionAider(final String user, final String group, final int mode) { + super(user, group, mode); + } + + @Override + public void read(final VariableByteInput istream) throws IOException { + // no-op + } + + @Override + public void setGroup(final Group group) { + // no-op + } + + @Override + public void setGroup(final String group) { + // no-op + } + + @Override + public void setGroup(final int id) { + // no-op + } + + @Override + public void setGroupFrom(final Permission other) { + // no-op + } + + @Override + public void setGroupMode(final int groupMode) { + // no-op + } + + @Override + public void setMode(final int mode) { + // no-op + } + + @Override + public void setOtherMode(final int otherMode) { + // no-op + } + + @Override + public void setOwner(final int id) { + // no-op + } + + @Override + public void setOwner(final Account user) { + // no-op + } + + @Override + public void setOwner(final String user) { + super.setOwner(user); + } + + @Override + public void setOwnerMode(final int ownerMode) { + // no-op + } + + @Override + public void setSetGid(final boolean setGid) { + // no-op + } + + @Override + public void setSetUid(final boolean setUid) { + // no-op + } + + @Override + public void setSticky(final boolean sticky) { + // no-op + } +} diff --git a/exist-core/src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java b/exist-core/src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java index 23a5505ed5..36592c9cdf 100644 --- a/exist-core/src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java +++ b/exist-core/src/main/java/org/exist/security/internal/aider/UnixStylePermissionAider.java @@ -360,7 +360,31 @@ public static UnixStylePermissionAider fromString(final String modeStr) throws S @Override public boolean validate(final Subject user, final int mode) { - throw new UnsupportedOperationException("Validation of Permission Aider is unsupported"); + + //group dba has full access + if (user.hasDbaRole()) { + return true; + } + + //check owner + if (user.getId() == owner.getId()) { //check owner + return (mode & ((this.mode >>> 6) & 7)) == mode; //check owner mode + } + + //check group + final int[] userGroupIds = user.getGroupIds(); + for(final int userGroupId : userGroupIds) { + if (userGroupId == ownerGroup.getId()) { + return (mode & ((this.mode >>> 3) & 7)) == mode; + } + } + + //check other + if ((mode & (this.mode & 7)) == mode) { + return true; + } + + return false; } @Override diff --git a/exist-core/src/main/java/org/exist/source/AbstractSource.java b/exist-core/src/main/java/org/exist/source/AbstractSource.java index ae627275a7..29a1fe1206 100644 --- a/exist-core/src/main/java/org/exist/source/AbstractSource.java +++ b/exist-core/src/main/java/org/exist/source/AbstractSource.java @@ -52,10 +52,10 @@ import net.jpountz.xxhash.XXHash64; import net.jpountz.xxhash.XXHashFactory; -import org.exist.EXistException; import org.exist.dom.QName; -import org.exist.security.PermissionDeniedException; -import org.exist.security.Subject; +import org.exist.security.Permission; +import org.exist.security.SecurityManager; +import org.exist.security.internal.aider.ImmutableUnixStylePermissionAider; import org.exist.xquery.XPathException; import org.exist.xquery.parser.DeclScanner; import org.exist.xquery.parser.XQueryLexer; @@ -71,6 +71,8 @@ */ public abstract class AbstractSource implements Source { + private static final Permission ALLOW_ALL_PERMISSIONS = new ImmutableUnixStylePermissionAider(SecurityManager.DBA_USER, SecurityManager.DBA_GROUP, 0777); + private final long key; protected AbstractSource(final long key) { @@ -87,11 +89,6 @@ public Charset getEncoding() throws IOException { return null; } - @Deprecated - public void validate(final Subject subject, final int perm) throws PermissionDeniedException, EXistException { - // no-op - } - @Override public boolean equals(final Object obj) { if (obj != null && obj instanceof Source) { @@ -174,6 +171,11 @@ public String pathOrContentOrShortIdentifier() { return str; } + @Override + public Permission getPermissions() throws IOException { + return ALLOW_ALL_PERMISSIONS; + } + protected static final long XXHASH64_SEED = 0x79742bc8; protected static long hashKey(final String key) { return hashKey(key.getBytes(UTF_8)); diff --git a/exist-core/src/main/java/org/exist/source/BinarySource.java b/exist-core/src/main/java/org/exist/source/BinarySource.java index 4a5b13a5ff..f0a4ea9b74 100644 --- a/exist-core/src/main/java/org/exist/source/BinarySource.java +++ b/exist-core/src/main/java/org/exist/source/BinarySource.java @@ -21,8 +21,6 @@ */ package org.exist.source; -import org.exist.security.Subject; -import org.exist.storage.DBBroker; import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream; import java.io.*; @@ -54,12 +52,7 @@ public String type() { } @Override - public Validity isValid(final DBBroker broker) { - return Source.Validity.VALID; - } - - @Override - public Validity isValid(final Source other) { + public Validity isValid() { return Source.Validity.VALID; } diff --git a/exist-core/src/main/java/org/exist/source/DBSource.java b/exist-core/src/main/java/org/exist/source/DBSource.java index 785743114c..1c54482431 100644 --- a/exist-core/src/main/java/org/exist/source/DBSource.java +++ b/exist-core/src/main/java/org/exist/source/DBSource.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,13 +47,13 @@ import java.io.*; +import org.exist.EXistException; import org.exist.dom.persistent.BinaryDocument; import org.exist.dom.QName; import org.exist.dom.persistent.LockedDocument; import org.exist.security.Permission; import org.exist.security.PermissionDeniedException; -import org.exist.security.Subject; -import org.exist.security.internal.aider.UnixStylePermissionAider; +import org.exist.storage.BrokerPool; import org.exist.storage.DBBroker; import org.exist.storage.lock.Lock.LockMode; import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; @@ -40,20 +64,24 @@ /** * Source implementation that reads from a binary resource * stored in the database. + * + * Should be used under a lock on the document, otherwise if + * no document is available, or only a lock for reads is required + * choose {@link DbUriSource} instead. * * @author wolf */ -public class DBSource extends AbstractSource { +public class DBSource extends AbstractSource implements DbStoreSource { private final BinaryDocument doc; private final long lastModified; private String encoding = UTF_8.name(); private final boolean checkEncoding; - private final DBBroker broker; + private final BrokerPool brokerPool; - public DBSource(final DBBroker broker, final BinaryDocument doc, final boolean checkXQEncoding) { + public DBSource(final BrokerPool brokerPool, final BinaryDocument doc, final boolean checkXQEncoding) { super(hashKey(doc.getURI().toString())); - this.broker = broker; + this.brokerPool = brokerPool; this.doc = doc; this.lastModified = doc.getLastModified(); this.checkEncoding = checkXQEncoding; @@ -69,6 +97,7 @@ public String type() { return "DB"; } + @Override public XmldbURI getDocumentPath() { return doc.getURI(); } @@ -78,9 +107,10 @@ public long getLastModified() { } @Override - public Validity isValid(final DBBroker broker) { + public Validity isValid() { Validity result; - try (final LockedDocument lockedDoc = broker.getXMLResource(doc.getURI(), LockMode.READ_LOCK)) { + try (final DBBroker broker = brokerPool.getBroker(); + final LockedDocument lockedDoc = broker.getXMLResource(doc.getURI(), LockMode.READ_LOCK)) { if (lockedDoc == null) { result = Validity.INVALID; } else if(lockedDoc.getDocument().getLastModified() > lastModified) { @@ -88,40 +118,34 @@ public Validity isValid(final DBBroker broker) { } else { result = Validity.VALID; } - } catch (final PermissionDeniedException pde) { + } catch (final EXistException | PermissionDeniedException pde) { result = Validity.INVALID; } return result; } - @Override - public Validity isValid(final Source other) { - final Validity result; - if (!(other instanceof DBSource)) { - result = Validity.INVALID; - } else if (((DBSource)other).getLastModified() > lastModified) { - result = Validity.INVALID; - } else { - result = Validity.VALID; - } - - return result; - } - @Override public Reader getReader() throws IOException { - final InputStream is = broker.getBinaryResource(doc); - final BufferedInputStream bis = new BufferedInputStream(is); - bis.mark(64); - checkEncoding(bis); - bis.reset(); - return new InputStreamReader(bis, encoding); + try (final DBBroker broker = brokerPool.getBroker()) { + final InputStream is = broker.getBinaryResource(doc); + final BufferedInputStream bis = new BufferedInputStream(is); + bis.mark(64); + checkEncoding(bis); + bis.reset(); + return new InputStreamReader(bis, encoding); + } catch (final EXistException e) { + throw new IOException(e.getMessage(), e); + } } @Override public InputStream getInputStream() throws IOException { - return broker.getBinaryResource(doc); + try (final DBBroker broker = brokerPool.getBroker()) { + return broker.getBinaryResource(doc); + } catch (final EXistException e) { + throw new IOException(e.getMessage(), e); + } } @Override @@ -131,20 +155,26 @@ public String getContent() throws IOException { throw new IOException("Resource too big to be read using this method."); } - try (final InputStream raw = broker.getBinaryResource(doc); + try (final DBBroker broker = brokerPool.getBroker(); + final InputStream raw = broker.getBinaryResource(doc); final UnsynchronizedByteArrayOutputStream buf = new UnsynchronizedByteArrayOutputStream((int)binaryLength)) { buf.write(raw); try (final InputStream is = buf.toInputStream()) { checkEncoding(is); return buf.toString(encoding); } + } catch (final EXistException e) { + throw new IOException(e.getMessage(), e); } } @Override public QName isModule() throws IOException { - try (final InputStream is = broker.getBinaryResource(doc)) { + try (final DBBroker broker = brokerPool.getBroker(); + final InputStream is = broker.getBinaryResource(doc)) { return getModuleDecl(is); + } catch (final EXistException e) { + throw new IOException(e.getMessage(), e); } } @@ -162,51 +192,7 @@ public String toString() { return doc.getDocumentURI(); } - /** - * Check: has current subject requested permissions for this resource? - * - * @param mode The requested mode - * @throws PermissionDeniedException if user has not sufficient rights - * - * @deprecated These security checks should be done by the caller - */ - @Deprecated - public void validate(final int mode) throws PermissionDeniedException { - //TODO(AR) This check should not even be here! Its up to the database to refuse access not requesting source - final Subject subject = broker.getCurrentSubject(); - if (subject != null) { - doValidation(subject, mode); - } - } - - /** - * Check: has subject requested permissions for this resource? - * - * @param subject The subject - * @param mode The requested mode - * @throws PermissionDeniedException if user has not sufficient rights - * - * @deprecated These security checks should be done by the caller - */ @Override - @Deprecated - public void validate(final Subject subject, final int mode) throws PermissionDeniedException { - //TODO(AR) This check should not even be here! Its up to the database to refuse access not requesting source - if (subject == null) { - final String modeStr = new UnixStylePermissionAider(mode).toString(); - throw new PermissionDeniedException("Subject not given for checking '" + modeStr + "' access to resource '" + doc.getURI() + "'."); - } else { - doValidation(subject, mode); - } - } - - private void doValidation(final Subject subject, final int mode) throws PermissionDeniedException { - if (!doc.getPermissions().validate(subject, mode)) { - final String modeStr = new UnixStylePermissionAider(mode).toString(); - throw new PermissionDeniedException("Subject '" + subject.getName() + "' does not have '" + modeStr + "' access to resource '" + doc.getURI() + "'."); - } - } - public Permission getPermissions() { return doc.getPermissions(); } diff --git a/exist-core/src/main/java/org/exist/source/DbStoreSource.java b/exist-core/src/main/java/org/exist/source/DbStoreSource.java new file mode 100644 index 0000000000..3041bb7922 --- /dev/null +++ b/exist-core/src/main/java/org/exist/source/DbStoreSource.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2014, Evolved Binary Ltd + * + * This file was originally ported from FusionDB to Elemental by + * Evolved Binary, for the benefit of the Elemental Open Source community. + * Only the ported code as it appears in this file, at the time that + * it was contributed to Elemental, was re-licensed under The GNU + * Lesser General Public License v2.1 only for use in Elemental. + * + * This license grant applies only to a snapshot of the code as it + * appeared when ported, it does not offer or infer any rights to either + * updates of this source code or access to the original source code. + * + * The GNU Lesser General Public License v2.1 only license follows. + * + * ===================================================================== + * + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.source; + +import org.exist.xmldb.XmldbURI; + +public interface DbStoreSource extends Source { + + /** + * Get the database URI of the document backing this source. + * + * @return the URI of the document backing this source. + */ + XmldbURI getDocumentPath(); +} diff --git a/exist-core/src/main/java/org/exist/source/DbUriSource.java b/exist-core/src/main/java/org/exist/source/DbUriSource.java new file mode 100644 index 0000000000..8d2a1510e2 --- /dev/null +++ b/exist-core/src/main/java/org/exist/source/DbUriSource.java @@ -0,0 +1,513 @@ +/* + * Copyright (C) 2014, Evolved Binary Ltd + * + * This file was originally ported from FusionDB to Elemental by + * Evolved Binary, for the benefit of the Elemental Open Source community. + * Only the ported code as it appears in this file, at the time that + * it was contributed to Elemental, was re-licensed under The GNU + * Lesser General Public License v2.1 only for use in Elemental. + * + * This license grant applies only to a snapshot of the code as it + * appeared when ported, it does not offer or infer any rights to either + * updates of this source code or access to the original source code. + * + * The GNU Lesser General Public License v2.1 only license follows. + * + * ===================================================================== + * + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.source; + +import java.io.*; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.Properties; + +import com.evolvedbinary.j8fu.function.*; +import org.exist.EXistException; +import org.exist.dom.QName; +import org.exist.dom.persistent.BinaryDocument; +import org.exist.dom.persistent.DocumentImpl; +import org.exist.dom.persistent.LockedDocument; +import org.exist.security.Permission; +import org.exist.security.PermissionDeniedException; +import org.exist.security.Subject; +import org.exist.storage.BrokerPool; +import org.exist.storage.DBBroker; +import org.exist.storage.lock.Lock.LockMode; +import org.exist.storage.serializers.Serializer; +import org.exist.storage.txn.TransactionException; +import org.exist.storage.txn.Txn; +import org.exist.util.io.InputStreamUtil; +import org.exist.util.io.TemporaryFileManager; +import org.exist.util.serializer.SAXSerializer; +import org.exist.util.serializer.SerializerPool; +import org.exist.xmldb.XmldbURI; +import org.xml.sax.SAXException; + +import javax.annotation.Nullable; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Source implementation that given a URI reads from a resource stored in the database. + * + * Read locks are only taken on the document + * during {@link #from(BrokerPool, XmldbURI, boolean, boolean)}, + * {@link #isValid()}, {@link #getPermissions()}, + * and {@link #getContent()}. + * + * Calling {@link #getInputStream()} or {@link #getReader()} witll + * return an object that holds a read lock on the document until + * {@link AutoCloseable#close()} is called on it. + * + * @author Adam Retter. + */ +public class DbUriSource extends AbstractSource implements DbStoreSource { + + private final BrokerPool brokerPool; + private final Optional dbSubject; + private final XmldbURI docUri; + private String encoding = UTF_8.name(); + private final boolean checkEncoding; + private final long docLastModified; + private final boolean errorOnModified; + + private DbUriSource(final BrokerPool brokerPool, final Optional dbSubject, final boolean checkXQEncoding, final XmldbURI docUri, final long docLastModified, final boolean errorOnModified) { + super(hashKey(docUri.getCollectionPath())); + this.brokerPool = brokerPool; + this.dbSubject = dbSubject; + this.checkEncoding = checkXQEncoding; + this.docUri = docUri; + this.docLastModified = docLastModified; + this.errorOnModified = errorOnModified; + } + + /** + * Create a new DbUriSource. + * + * @param brokerPool the Broker Pool. + * @param docUri URI to the document in the database. + * @param checkXQEncoding true if the encoding should be checked from the XQuery. + * @param errorOnModified when true and error is raised if the document was modified since the source was created. + * + * @throws NoSuchDocumentException the provided docUri does not exist in the database. + * @throws EXistException an error occurred whilst accessing the database. + * @throws PermissionDeniedException the calling user has insufficient permissions to access the document in the database. + */ + public static DbUriSource from(final BrokerPool brokerPool, final XmldbURI docUri, final boolean checkXQEncoding, final boolean errorOnModified) throws NoSuchDocumentException, EXistException, PermissionDeniedException { + return from(brokerPool, null, docUri, checkXQEncoding, errorOnModified); + } + + /** + * Create a new DbUriSource. + * + * @param brokerPool the Broker Pool. + * @param dbSubject the database user for database operations, or null to use the current subject. + * @param docUri URI to the document in the database. + * @param checkXQEncoding true if the encoding should be checked from the XQuery. + * @param errorOnModified when true and error is raised if the document was modified since the source was created. + * + * @throws NoSuchDocumentException the provided docUri does not exist in the database. + * @throws EXistException an error occurred whilst accessing the database. + * @throws PermissionDeniedException the calling user has insufficient permissions to access the document in the database. + */ + public static DbUriSource from(final BrokerPool brokerPool, @Nullable final Subject dbSubject, final XmldbURI docUri, final boolean checkXQEncoding, final boolean errorOnModified) throws NoSuchDocumentException, EXistException, PermissionDeniedException { + final long docLastModified = DbUriSource.readDocument(brokerPool, Optional.ofNullable(dbSubject), docUri, (broker, transaction, doc) -> { + if (doc == null) { + throw new NoSuchDocumentException("No such document: " + docUri.getCollectionPath()); + } + return doc.getLastModified(); + }); + + return new DbUriSource(brokerPool, Optional.ofNullable(dbSubject), checkXQEncoding, docUri, docLastModified, errorOnModified); + } + + /** + * Create a new DbUriSource. + * + * @param brokerPool the Broker Pool. + * @param doc the document in the database. + * @param checkXQEncoding true if the encoding should be checked from the XQuery. + * @param errorOnModified when true and error is raised if the document was modified since the source was created. + */ + public static DbUriSource from(final BrokerPool brokerPool, final DocumentImpl doc, final boolean checkXQEncoding, final boolean errorOnModified) { + return from(brokerPool, null, doc, checkXQEncoding, errorOnModified); + } + + /** + * Create a new DbUriSource. + * + * @param brokerPool the Broker Pool. + * @param dbSubject the database user for database operations, or null to use the current subject. + * @param doc the document in the database. + * @param checkXQEncoding true if the encoding should be checked from the XQuery. + * @param errorOnModified when true and error is raised if the document was modified since the source was created. + */ + public static DbUriSource from(final BrokerPool brokerPool, @Nullable final Subject dbSubject, final DocumentImpl doc, final boolean checkXQEncoding, final boolean errorOnModified) { + return new DbUriSource(brokerPool, Optional.ofNullable(dbSubject), checkXQEncoding, doc.getURI(), doc.getLastModified(), errorOnModified); + } + + @Override + public String path() { + return docUri.getCollectionPath(); + } + + @Override + public String type() { + return "DB"; + } + + @Override + public XmldbURI getDocumentPath() { + return docUri; + } + + @Override + public Validity isValid() { + try { + return readDocument((broker, transaction, doc) -> { + if (doc == null) { + return Validity.INVALID; + + } else if (doc.getLastModified() > docLastModified) { + return Validity.INVALID; + + } else { + return Validity.VALID; + } + }); + } catch (final EXistException | PermissionDeniedException pde) { + return Validity.INVALID; + } + } + + @Override + public Reader getReader() throws IOException { + final BufferedInputStream bis = new BufferedInputStream(getInputStream()); + try { + bis.mark(128); + checkEncoding(bis); + bis.reset(); + return new InputStreamReader(bis, encoding); + } catch (final IOException e) { + bis.close(); + throw e; + } + } + + @Override + public InputStream getInputStream() throws IOException { + @Nullable DBBroker broker = null; + @Nullable Txn transaction = null; + @Nullable LockedDocument lockedDoc = null; + + try { + broker = brokerPool.get(dbSubject); + transaction = brokerPool.getTransactionManager().beginTransaction(); + lockedDoc = broker.getXMLResource(docUri, LockMode.READ_LOCK); + + if (lockedDoc == null) { + throw new NoSuchDocumentException("No such document: " + docUri.getCollectionPath()); + } + + final DocumentImpl doc = lockedDoc.getDocument(); + if (doc.getLastModified() > docLastModified && errorOnModified) { + throw new IOException("Document has been modified since DbUriSource was created: " + docUri.getCollectionPath()); + } + + if (doc instanceof BinaryDocument) { + // Get InputStream of Binary document + return BinaryDocumentInputStream.of(broker, transaction, lockedDoc); + + } else { + // Get InputStream of XML document + return XMLDocumentInputStream.of(broker, transaction, lockedDoc); + } + } catch (final EXistException | PermissionDeniedException | IOException e) { + if (lockedDoc != null) { + lockedDoc.close(); + } + + if (transaction != null) { + transaction.abort(); + transaction.close(); + } + + if (broker != null) { + broker.close(); + } + + if (e instanceof IOException) { + throw (IOException) e; + } else { + throw new IOException(e.getMessage(), e); + } + } + } + + @Override + public String getContent() throws IOException { + try (final InputStream is = new BufferedInputStream(getInputStream())) { + return InputStreamUtil.readString(is, Charset.forName(encoding)); + } + } + + @Override + public QName isModule() throws IOException { + try (final InputStream is = getInputStream()) { + return getModuleDecl(is); + } + } + + private void checkEncoding(final InputStream is) { + if (checkEncoding) { + final String checkedEnc = guessXQueryEncoding(is); + if(checkedEnc != null) { + encoding = checkedEnc; + } + } + } + + @Override + public String toString() { + return docUri.getCollectionPath(); + } + + @Override + public Permission getPermissions() throws IOException { + try { + return readDocument((broker, transaction, doc) -> doc.getPermissions()); + } catch (final EXistException | PermissionDeniedException e) { + throw new IOException(e); + } + } + + @Override + public int hashCode() { + return getDocumentPath().hashCode(); + } + + private T readDocument(final TriFunctionE documentOp) throws E, EXistException, PermissionDeniedException { + return readDocument(brokerPool, dbSubject, docUri, documentOp); + } + + private static T readDocument(final BrokerPool brokerPool, final Optional dbSubject, final XmldbURI docUri, final TriFunctionE documentOp) throws E, EXistException, PermissionDeniedException { + try (final DBBroker broker = brokerPool.get(dbSubject); + final Txn transaction = brokerPool.getTransactionManager().beginTransaction(); + final LockedDocument lockedDoc = broker.getXMLResource(docUri, LockMode.READ_LOCK)) { + DocumentImpl doc = null; + if (lockedDoc != null) { + doc = lockedDoc.getDocument(); + } + + final T result = documentOp.apply(broker, transaction, doc); + + transaction.commit(); + + return result; + } + } + + public static class NoSuchDocumentException extends EXistException { + public NoSuchDocumentException(final String message) { + super(message); + } + } + + /** + * An InputStream for a binary document that takes ownership of the DBBroker, Txn, and Document Lock. + */ + public static class BinaryDocumentInputStream extends FilterInputStream { + + private final DBBroker broker; + private final Txn transaction; + private final LockedDocument lockedDocument; + private boolean closed = false; + + private BinaryDocumentInputStream(final DBBroker broker, final Txn transaction, final LockedDocument lockedDocument, final InputStream is) { + super(is); + this.broker = broker; + this.transaction = transaction; + this.lockedDocument = lockedDocument; + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + + @Nullable IOException ioe = null; + try { + super.close(); + } catch (final IOException e) { + ioe = e; + } + + lockedDocument.close(); + + @Nullable TransactionException te = null; + try { + if (ioe != null) { + transaction.abort(); + } else { + transaction.commit(); + } + transaction.close(); + } catch (final TransactionException e) { + te = e; + } + + broker.close(); + + closed = true; + + if (ioe != null) { + throw ioe; + } + + if (te != null) { + throw new IOException(te); + } + } + + public static BinaryDocumentInputStream of(final DBBroker broker, final Txn transaction, final LockedDocument lockedDocument) throws IOException { + final DocumentImpl doc = lockedDocument.getDocument(); + if (!(doc instanceof BinaryDocument)) { + throw new IllegalArgumentException("Expected binary document but received: " + lockedDocument.getClass().getSimpleName()); + } + + final InputStream is = broker.getBinaryResource(transaction, (BinaryDocument) doc); + return new BinaryDocumentInputStream(broker, transaction, lockedDocument, is); + } + } + + /** + * An InputStream for an XML document that takes ownership of the DBBroker, Txn, Document Lock, and a Temporary File. + */ + public static class XMLDocumentInputStream extends FilterInputStream { + + private final DBBroker broker; + private final Txn transaction; + private final LockedDocument lockedDocument; + private final TemporaryFileManager temporaryFileManager; + private final Path temporaryFile; + private boolean closed = false; + + private XMLDocumentInputStream(final DBBroker broker, final Txn transaction, final LockedDocument lockedDocument, final TemporaryFileManager temporaryFileManager, final Path temporaryFile, final InputStream is) { + super(is); + this.broker = broker; + this.transaction = transaction; + this.lockedDocument = lockedDocument; + this.temporaryFileManager = temporaryFileManager; + this.temporaryFile = temporaryFile; + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + + @Nullable IOException ioe = null; + try { + super.close(); + } catch (final IOException e) { + ioe = e; + } + + temporaryFileManager.returnTemporaryFile(temporaryFile); + + lockedDocument.close(); + + @Nullable TransactionException te = null; + try { + if (ioe != null) { + transaction.abort(); + } else { + transaction.commit(); + } + transaction.close(); + } catch (final TransactionException e) { + te = e; + } + + broker.close(); + + closed = true; + + if (ioe != null) { + throw ioe; + } + + if (te != null) { + throw new IOException(te); + } + } + + public static XMLDocumentInputStream of(final DBBroker broker, final Txn transaction, final LockedDocument lockedDocument) throws IOException { + final DocumentImpl doc = lockedDocument.getDocument(); + if (doc instanceof BinaryDocument) { + throw new IllegalArgumentException("Expected XML document but received: " + lockedDocument.getClass().getSimpleName()); + } + + // Serialize XML to temporary file + final TemporaryFileManager temporaryFileManager = TemporaryFileManager.getInstance(); + @Nullable Path temporaryFile = temporaryFileManager.getTemporaryFile(); + try { + + final Properties outputProperties = new Properties(); + outputProperties.setProperty("method", "xml"); + outputProperties.setProperty("media-type", "application/xml; charset=UTF-8"); + outputProperties.setProperty("indent", "yes"); + outputProperties.setProperty("omit-xml-declaration", "yes"); + + final Serializer serializer = broker.borrowSerializer(); + serializer.setProperties(outputProperties); + try { + final SAXSerializer sax = (SAXSerializer) SerializerPool.getInstance().borrowObject(SAXSerializer.class); + try (final OutputStream os = new BufferedOutputStream(Files.newOutputStream(temporaryFile)); + final Writer writer = new OutputStreamWriter(os, UTF_8)) { + sax.setOutput(writer, outputProperties); + serializer.setSAXHandlers(sax, sax); + + serializer.toSAX(doc); + + } finally { + SerializerPool.getInstance().returnObject(sax); + } + } finally { + broker.returnSerializer(serializer); + } + } catch (final SAXException e) { + if (temporaryFile != null) { + temporaryFileManager.returnTemporaryFile(temporaryFile); + } + } + + final InputStream is = Files.newInputStream(temporaryFile); + return new XMLDocumentInputStream(broker, transaction, lockedDocument, temporaryFileManager, temporaryFile, is); + } + } +} diff --git a/exist-core/src/main/java/org/exist/source/FileSource.java b/exist-core/src/main/java/org/exist/source/FileSource.java index 469771c29b..6241a56871 100644 --- a/exist-core/src/main/java/org/exist/source/FileSource.java +++ b/exist-core/src/main/java/org/exist/source/FileSource.java @@ -33,8 +33,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.exist.dom.QName; -import org.exist.security.Subject; -import org.exist.storage.DBBroker; /** * A source implementation reading from the path system. @@ -82,7 +80,7 @@ public Path getPath() { } @Override - public Validity isValid(final DBBroker broker) { + public Validity isValid() { final long currentLastModified = lastModifiedSafe(path); if (currentLastModified == -1 || currentLastModified > lastModified) { return Validity.INVALID; @@ -91,11 +89,6 @@ public Validity isValid(final DBBroker broker) { } } - @Override - public Validity isValid(final Source other) { - return Validity.INVALID; - } - @Override public Reader getReader() throws IOException { checkEncoding(); diff --git a/exist-core/src/main/java/org/exist/source/Source.java b/exist-core/src/main/java/org/exist/source/Source.java index aba6b11e60..e6d1e6323d 100644 --- a/exist-core/src/main/java/org/exist/source/Source.java +++ b/exist-core/src/main/java/org/exist/source/Source.java @@ -50,11 +50,8 @@ import java.io.InputStream; import java.nio.charset.Charset; -import org.exist.EXistException; import org.exist.dom.QName; -import org.exist.security.PermissionDeniedException; -import org.exist.security.Subject; -import org.exist.storage.DBBroker; +import org.exist.security.Permission; import javax.annotation.Nullable; @@ -70,8 +67,7 @@ public interface Source { enum Validity { VALID, - INVALID, - UNKNOWN + INVALID } /** @@ -98,32 +94,10 @@ enum Validity { /** * Is this source object still valid? - * - * Returns {@link Validity#UNKNOWN} if the validity of - * the source cannot be determined. - * - * The {@link DBBroker} parameter is required by - * some implementations as they have to read - * resources from the database. - * - * @param broker the broker + * * @return Validity of the source object */ - Validity isValid(DBBroker broker); - - /** - * Checks if the source object is still valid - * by comparing it to another version of the - * same source. It depends on the concrete - * implementation how the sources are compared. - * - * Use this method if {@link #isValid(DBBroker)} - * return {@link Validity#UNKNOWN}. - * - * @param other source - * @return Validity of the other source object - */ - Validity isValid(Source other); + Validity isValid(); /** * Returns a {@link Reader} to read the contents @@ -160,17 +134,14 @@ enum Validity { @Nullable Charset getEncoding() throws IOException; /** - * Check: has subject requested permissions for this resource? + * Get the permissions on the resource backing this source. * - * @param subject The subject - * @param perm The requested permissions - * @throws PermissionDeniedException if user has not sufficient rights + * @return the permissions on the resource backing this source. * @throws EXistException if an error occurs acessing the database * - * @deprecated These security checks only apply to {@link DBSource} and should be done by the caller + * @throws IOException if the permissions cannot be retrieved. */ - @Deprecated - void validate(Subject subject, int perm) throws PermissionDeniedException, EXistException; + Permission getPermissions() throws IOException; /** * Check if the source is an XQuery module. If it is, return a QName containing diff --git a/exist-core/src/main/java/org/exist/source/SourceFactory.java b/exist-core/src/main/java/org/exist/source/SourceFactory.java index 1aef1b22a0..a6cb344f65 100644 --- a/exist-core/src/main/java/org/exist/source/SourceFactory.java +++ b/exist-core/src/main/java/org/exist/source/SourceFactory.java @@ -58,17 +58,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.exist.EXistException; -import org.exist.dom.persistent.BinaryDocument; -import org.exist.dom.persistent.DocumentImpl; -import org.exist.dom.persistent.LockedDocument; import org.exist.security.PermissionDeniedException; import org.exist.storage.BrokerPool; import org.exist.storage.DBBroker; -import org.exist.storage.lock.Lock.LockMode; -import org.exist.storage.serializers.Serializer; import org.exist.util.FileUtils; import org.exist.xmldb.XmldbURI; -import org.xml.sax.SAXException; import javax.annotation.Nullable; @@ -204,27 +198,13 @@ private static Source getSource_fromClasspath(final String contextPath, final St * @return the source, or null if there is no such resource in the db indicated by {@code path}. */ private static @Nullable Source getSource_fromDb(final DBBroker broker, final XmldbURI path) throws PermissionDeniedException, IOException { - Source source = null; - try(final LockedDocument lockedResource = broker.getXMLResource(path, LockMode.READ_LOCK)) { - if (lockedResource != null) { - final DocumentImpl resource = lockedResource.getDocument(); - if (resource.getResourceType() == DocumentImpl.BINARY_FILE) { - source = new DBSource(broker, (BinaryDocument) resource, true); - } else { - final Serializer serializer = broker.borrowSerializer(); - try { - // XML document: serialize to string source so it can be read as a stream - // by fn:unparsed-text and friends - source = new StringSource(serializer.serialize(resource)); - } catch (final SAXException e) { - throw new IOException(e.getMessage()); - } finally { - broker.returnSerializer(serializer); - } - } - } + try { + return DbUriSource.from(broker.getBrokerPool(), path, true, false); + } catch (final DbUriSource.NoSuchDocumentException e) { + return null; + } catch (final EXistException e) { + throw new IOException(e); } - return source; } /** diff --git a/exist-core/src/main/java/org/exist/source/StringSource.java b/exist-core/src/main/java/org/exist/source/StringSource.java index a772c6d601..0845b63ed7 100644 --- a/exist-core/src/main/java/org/exist/source/StringSource.java +++ b/exist-core/src/main/java/org/exist/source/StringSource.java @@ -25,8 +25,6 @@ import java.io.Reader; import java.io.StringReader; -import org.exist.security.Subject; -import org.exist.storage.DBBroker; import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream; import static java.nio.charset.StandardCharsets.UTF_8; @@ -56,12 +54,7 @@ public String type() { } @Override - public Validity isValid(final DBBroker broker) { - return Source.Validity.VALID; - } - - @Override - public Validity isValid(final Source other) { + public Validity isValid() { return Source.Validity.VALID; } diff --git a/exist-core/src/main/java/org/exist/source/URLSource.java b/exist-core/src/main/java/org/exist/source/URLSource.java index 801d947d71..c305d0a772 100644 --- a/exist-core/src/main/java/org/exist/source/URLSource.java +++ b/exist-core/src/main/java/org/exist/source/URLSource.java @@ -48,7 +48,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.exist.dom.QName; -import org.exist.storage.DBBroker; import xyz.elemental.mediatype.MediaType; import java.io.IOException; @@ -124,7 +123,7 @@ private long getLastModification() { } @Override - public Validity isValid(final DBBroker broker) { + public Validity isValid() { final long modified = getLastModification(); final Validity validity; if (modified == 0 || modified > lastModified) { @@ -136,11 +135,6 @@ public Validity isValid(final DBBroker broker) { return validity; } - @Override - public Validity isValid(final Source other) { - return Validity.INVALID; - } - @Override public Charset getEncoding() throws IOException { if (connection == null) { diff --git a/exist-core/src/main/java/org/exist/storage/XQueryPool.java b/exist-core/src/main/java/org/exist/storage/XQueryPool.java index 19be2a931b..12c2e704b9 100644 --- a/exist-core/src/main/java/org/exist/storage/XQueryPool.java +++ b/exist-core/src/main/java/org/exist/storage/XQueryPool.java @@ -32,6 +32,7 @@ */ package org.exist.storage; +import java.io.IOException; import java.text.NumberFormat; import java.util.ArrayDeque; import java.util.Deque; @@ -43,7 +44,6 @@ import org.apache.logging.log4j.Logger; import org.exist.security.Permission; import org.exist.security.PermissionDeniedException; -import org.exist.source.DBSource; import org.exist.source.Source; import org.exist.util.Configuration; import org.exist.util.Holder; @@ -162,7 +162,7 @@ public CompiledXQuery borrowCompiledXQuery(final DBBroker broker, final Source s return null; } - if (!isCompiledQueryValid(broker, source, firstCompiledXQuery)) { + if (!isCompiledQueryValid(firstCompiledXQuery)) { if (LOG.isDebugEnabled()) { LOG.debug("{} is invalid, removing from XQuery Pool...", source.pathOrShortIdentifier()); } @@ -183,8 +183,12 @@ public CompiledXQuery borrowCompiledXQuery(final DBBroker broker, final Source s } //check execution permission - if (source instanceof DBSource) { - ((DBSource) source).validate(Permission.EXECUTE); + try { + if (!source.getPermissions().validate(broker.getCurrentSubject(), Permission.EXECUTE)) { + throw new PermissionDeniedException("Subject '" + broker.getCurrentSubject().getName() + "' does not have execute access to resource '" + source.pathOrShortIdentifier() + "'."); + } + } catch (final IOException e) { + throw new PermissionDeniedException("Subject '" + broker.getCurrentSubject().getName() + "' does not have execute access to resource '" + source.pathOrShortIdentifier() + "'.", e); } return borrowedCompiledQuery.value; @@ -193,22 +197,16 @@ public CompiledXQuery borrowCompiledXQuery(final DBBroker broker, final Source s /** * Determines if a compiled XQuery is still valid. * - * @param broker the database broker - * @param source the source of the query * @param compiledXQuery the compiled query * * @return true if the compiled query is still valid, false otherwise. */ - private static boolean isCompiledQueryValid(final DBBroker broker, final Source source, - final CompiledXQuery compiledXQuery) { + private static boolean isCompiledQueryValid(final CompiledXQuery compiledXQuery) { final Source cachedSource = compiledXQuery.getSource(); - Source.Validity validity = cachedSource.isValid(broker); - if (validity == Source.Validity.UNKNOWN) { - validity = cachedSource.isValid(source); - } + final Source.Validity validity = cachedSource.isValid(); - if (validity == Source.Validity.INVALID || validity == Source.Validity.UNKNOWN) { - return false; // returning null will remove the entry from the cache + if (validity == Source.Validity.INVALID) { + return false; // returning false will remove the entry from the cache } // the compiled query is no longer valid if one of the imported diff --git a/exist-core/src/main/java/org/exist/storage/lock/LockTable.java b/exist-core/src/main/java/org/exist/storage/lock/LockTable.java index abcc43782f..5e45b3b6a8 100644 --- a/exist-core/src/main/java/org/exist/storage/lock/LockTable.java +++ b/exist-core/src/main/java/org/exist/storage/lock/LockTable.java @@ -346,68 +346,72 @@ public Entry unmerge(final String id, final LockType lockType, final LockMode lo // optimistic read long stamp = entriesLock.tryOptimisticRead(); - Entry local = entries.get(key); - // if count is equal to 1 we can just remove from the list rather than decrementing - if (local.count == 1) { - final long writeStamp = entriesLock.tryConvertToWriteLock(stamp); - if (writeStamp != 0L) { - try { - entries.remove(local); - local.count--; - return local; - } finally { - entriesLock.unlockWrite(writeStamp); + @Nullable Entry local = entries.get(key); + if (local != null) { + // if count is equal to 1 we can just remove from the list rather than decrementing + if (local.count == 1) { + final long writeStamp = entriesLock.tryConvertToWriteLock(stamp); + if (writeStamp != 0L) { + try { + entries.remove(local); + local.count--; + return local; + } finally { + entriesLock.unlockWrite(writeStamp); + } } - } - } else { - if (entriesLock.validate(stamp)) { + } else { + if (entriesLock.validate(stamp)) { - // do the unmerge bit - if (local.stackTraces != null) { - local.stackTraces.remove(local.stackTraces.size() - 1); - } - local.count = local.count - 1; + // do the unmerge bit + if (local.stackTraces != null) { + local.stackTraces.remove(local.stackTraces.size() - 1); + } + local.count = local.count - 1; - //done - return local; + //done + return local; + } } } - // otherwise... pessimistic read - boolean mustRemove; + boolean mustRemove = false; stamp = entriesLock.readLock(); try { local = entries.get(key); - // if count is equal to 1 we can just remove from the list rather than decrementing - if (local.count == 1) { + if (local != null) { + // if count is equal to 1 we can just remove from the list rather than decrementing + if (local.count == 1) { + + final long writeStamp = entriesLock.tryConvertToWriteLock(stamp); + if (writeStamp != 0L) { + stamp = writeStamp; // NOTE: this causes the write lock to be released in the finally further down + entries.remove(local); + local.count--; + return local; + } - final long writeStamp = entriesLock.tryConvertToWriteLock(stamp); - if (writeStamp != 0L) { - stamp = writeStamp; // NOTE: this causes the write lock to be released in the finally further down - entries.remove(local); - local.count--; - return local; - } + } else { + // do the unmerge bit + if (local.stackTraces != null) { + local.stackTraces.remove(local.stackTraces.size() - 1); + } + local.count = local.count - 1; - } else { - // do the unmerge bit - if (local.stackTraces != null) { - local.stackTraces.remove(local.stackTraces.size() - 1); + //done + return local; } - local.count = local.count - 1; - //done - return local; + mustRemove = true; } - - mustRemove = true; } finally { entriesLock.unlock(stamp); } + // unable to remove by tryConvertToWriteLock above, so directly acquire write lock if (mustRemove) { stamp = entriesLock.writeLock(); diff --git a/exist-core/src/main/java/org/exist/storage/serializers/XIncludeFilter.java b/exist-core/src/main/java/org/exist/storage/serializers/XIncludeFilter.java index 38eb7414fe..ec44649943 100644 --- a/exist-core/src/main/java/org/exist/storage/serializers/XIncludeFilter.java +++ b/exist-core/src/main/java/org/exist/storage/serializers/XIncludeFilter.java @@ -46,6 +46,8 @@ package org.exist.storage.serializers; import com.evolvedbinary.j8fu.function.ConsumerE; +import com.evolvedbinary.j8fu.tuple.Tuple2; +import com.evolvedbinary.j8fu.tuple.Tuple3; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.exist.Namespaces; @@ -54,12 +56,13 @@ import org.exist.dom.persistent.DocumentImpl; import org.exist.dom.QName; import org.exist.dom.memtree.SAXAdapter; -import org.exist.security.Permission; +import org.exist.dom.persistent.LockedDocument; import org.exist.security.PermissionDeniedException; -import org.exist.source.DBSource; +import org.exist.source.DbUriSource; import org.exist.source.Source; import org.exist.source.StringSource; import com.evolvedbinary.j8fu.Either; +import org.exist.storage.lock.Lock; import org.exist.util.XMLReaderPool; import org.exist.util.serializer.AttrList; import org.exist.util.serializer.Receiver; @@ -73,13 +76,13 @@ import org.exist.xquery.value.Sequence; import org.exist.xquery.value.SequenceIterator; import org.exist.xquery.value.Type; +import org.jspecify.annotations.Nullable; import org.w3c.dom.Document; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.XMLReader; import xyz.elemental.mediatype.MediaType; -import javax.annotation.Nullable; import javax.xml.XMLConstants; import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; @@ -97,6 +100,7 @@ import java.util.Optional; import java.util.StringTokenizer; +import static com.evolvedbinary.j8fu.tuple.Tuple.Tuple; import static java.nio.charset.StandardCharsets.UTF_8; /** @@ -296,37 +300,178 @@ protected Optional processXInclude(final String href, String xpoi if (href == null) { throw new SAXException("No href attribute found in XInclude include element"); } + // save some settings DocumentImpl prevDoc = document; boolean createContainerElements = serializer.createContainerElements; serializer.createContainerElements = false; - //The following comments are the basis for possible external documents - XmldbURI docUri = null; + final Either, @Nullable XmldbURI, @Nullable MaybeLockedDocument>> errorOrXincludeDocAndParams = getXincludeDoc(href); + if (errorOrXincludeDocAndParams.isLeft()) { + final ResourceError resourceError = errorOrXincludeDocAndParams.left().get(); + return Optional.of(resourceError); + } + + final Source source; + final XmldbURI sourceUri; + @Nullable final Sequence contextSeq; + @Nullable final Map params; + + final Tuple3<@Nullable Map, @Nullable XmldbURI, @Nullable MaybeLockedDocument> xincludeDocAndParams = errorOrXincludeDocAndParams.right().get(); try { - docUri = XmldbURI.xmldbUriFor(href); - /* - if(!stylesheetUri.toCollectionPathURI().equals(stylesheetUri)) { - externalUri = stylesheetUri.getXmldbURI(); - } - */ - } catch (final URISyntaxException e) { - //could be an external URI! + @Nullable final Document xincludeDoc; + if (xincludeDocAndParams._3 != null) { + xincludeDoc = xincludeDocAndParams._3.getDocument(); + } else { + xincludeDoc = null; + } + @Nullable final XmldbURI xincludeDocUri = xincludeDocAndParams._2; + params = xincludeDocAndParams._1; + + /* if document has not been found and xpointer is + * null, throw an exception. If xpointer != null + * we retry below and interpret docName as + * a collection. + */ + if (xincludeDoc == null && xpointer == null) { + return Optional.of(new ResourceError("document " + xincludeDocUri + " not found")); + } + + /* Check if the document is a stored XQuery */ + final boolean xqueryDoc; + if (xincludeDoc instanceof BinaryDocument) { + xqueryDoc = MediaType.APPLICATION_XQUERY.equals(((BinaryDocument) xincludeDoc).getMediaType()); + } else { + xqueryDoc = false; + } + + if (xpointer == null && !xqueryDoc && xincludeDoc != null) { + // no xpointer found - just serialize the doc + if (xincludeDoc instanceof DocumentImpl) { + serializer.serializeToReceiver((DocumentImpl) xincludeDoc, false); + } else { + serializer.serializeToReceiver((org.exist.dom.memtree.DocumentImpl) xincludeDoc, false); + } + // restore settings + document = prevDoc; + serializer.createContainerElements = createContainerElements; + + return Optional.empty(); + } + + if (xpointer == null) { + source = DbUriSource.from(serializer.broker.getBrokerPool(), serializer.broker.getCurrentSubject(), (BinaryDocument) xincludeDoc, true, false); + sourceUri = ((DocumentImpl) xincludeDoc).getURI(); + } else { + try { + xpointer = checkNamespaces(xpointer); + } catch (final IllegalArgumentException e) { + LOG.warn("XPointer error: {}", e.getMessage(), e); + throw new SAXException("Error while processing XInclude expression: " + e.getMessage(), e); + } + source = new StringSource(xpointer); + sourceUri = xincludeDocUri; + } + + if (xincludeDoc instanceof org.exist.dom.memtree.DocumentImpl) { + contextSeq = (org.exist.dom.memtree.DocumentImpl) xincludeDoc; + } else { + contextSeq = null; + } + + } finally { + if (xincludeDocAndParams._3 != null) { + final MaybeLockedDocument maybeLockedDocument = xincludeDocAndParams._3; + maybeLockedDocument.close(); + } } + + // process the xpointer or the stored XQuery + try { + final String xpointerCopy = xpointer; + final ConsumerE setupXqueryContextPreCompilation = xqueryContext -> { + if (namespaces != null) { + xqueryContext.declareNamespaces(namespaces); + } + xqueryContext.declareNamespace("xinclude", Namespaces.XINCLUDE_NS); + + // Setup the HTTP context if known + if (serializer.httpContext != null) { + xqueryContext.setHttpContext(serializer.httpContext); + } + + if (xpointerCopy != null) { + xqueryContext.setStaticallyKnownDocuments(new XmldbURI[]{ sourceUri }); + } + }; + + final ConsumerE setupXqueryContextPreExecution = xqueryContext -> { + if (document != null) { + xqueryContext.declareVariable("xinclude:current-doc", true, document.getFileURI().toString()); + xqueryContext.declareVariable("xinclude:current-collection", true, document.getCollection().getURI().toString()); + } + + // pass parameters as variables + if (params != null) { + for (final Map.Entry entry : params.entrySet()) { + xqueryContext.declareVariable(entry.getKey(), true, entry.getValue()); + } + } + }; + + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(serializer.broker, source, xpointer != null, true, contextSeq, null, setupXqueryContextPreCompilation, setupXqueryContextPreExecution, null)) { + + final Sequence seq = queryResult.result; + + if (Type.subTypeOf(seq.getItemType(), Type.NODE)) { + if (LOG.isDebugEnabled()) { + LOG.debug("XPointer found: {}", seq.getItemCount()); + } + + NodeValue node; + for (final SequenceIterator i = seq.iterate(); i.hasNext(); ) { + node = (NodeValue) i.nextItem(); + serializer.serializeToReceiver(node, false); + } + } else { + String val; + for (int i = 0; i < seq.getItemCount(); i++) { + val = seq.itemAt(i).getStringValue(); + characters(val); + } + } + } + + } catch (final XPathException | IOException | PermissionDeniedException e) { + LOG.warn("XPointer error: {}", e.getMessage(), e); + throw new SAXException("Error while processing XInclude expression: " + e.getMessage(), e); + } + + // restore settings + document = prevDoc; + serializer.createContainerElements = createContainerElements; + + return Optional.empty(); + } + + private Either, @Nullable XmldbURI, @Nullable MaybeLockedDocument>> getXincludeDoc(final String href) throws SAXException { // parse the href attribute - LOG.debug("found href=\"{}\"", href); - //String xpointer = null; - //String docName = href; + if (LOG.isDebugEnabled()) { + LOG.debug("found href=\"{}\"", href); + } - final Map params; - DocumentImpl doc = null; - org.exist.dom.memtree.DocumentImpl memtreeDoc = null; - boolean xqueryDoc = false; + XmldbURI docUri = null; + try { + docUri = XmldbURI.xmldbUriFor(href); + } catch (final URISyntaxException e) { + // no-op... could be an external URI! + } + @Nullable final Map params; if (docUri != null) { final String fragment = docUri.getFragment(); - if (!(fragment == null || fragment.length() == 0)) { + if (!(fragment == null || fragment.isEmpty())) { throw new SAXException("Fragment identifiers must not be used in an xinclude href attribute. To specify an xpointer, use the xpointer attribute."); } @@ -358,163 +503,84 @@ protected Optional processXInclude(final String href, String xpoi // retrieve the document try { - doc = serializer.broker.getResource(docUri, Permission.READ); + return Either.Right(Tuple(params, docUri, MaybeLockedDocument.of(serializer.broker.getXMLResource(docUri, Lock.LockMode.READ_LOCK)))); } catch (final PermissionDeniedException e) { - return Optional.of(new ResourceError("Permission denied to read XInclude'd resource", e)); - } - - /* Check if the document is a stored XQuery */ - if (doc != null && doc.getResourceType() == DocumentImpl.BINARY_FILE) { - xqueryDoc = MediaType.APPLICATION_XQUERY.equals(doc.getMediaType()); + return Either.Left(new ResourceError("Permission denied to read XInclude'd resource", e)); } } else { params = null; } // The document could not be found: check if it points to an external resource - if (docUri == null || (doc == null && !docUri.isAbsolute())) { - try { - URI externalUri = new URI(href); - final String scheme = externalUri.getScheme(); - // If the URI has no scheme specified, - // we have to check if it is a relative path, and if yes, try to - // interpret it relative to the moduleLoadPath property of the current - // XQuery context. - if (scheme == null && moduleLoadPath != null) { - final String path = externalUri.getSchemeSpecificPart(); - Path f = Paths.get(path); - if (!f.isAbsolute()) { - if (moduleLoadPath.startsWith(XmldbURI.XMLDB_URI_PREFIX)) { - final XmldbURI parentUri = XmldbURI.create(moduleLoadPath); - docUri = parentUri.append(path); - doc = (DocumentImpl) serializer.broker.getXMLResource(docUri); - if (doc != null && !doc.getPermissions().validate(serializer.broker.getCurrentSubject(), Permission.READ)) { - throw new PermissionDeniedException("Permission denied to read XInclude'd resource"); - } - } else { - f = Paths.get(moduleLoadPath, path); - externalUri = f.toUri(); - } - } - } - if (doc == null) { - final Either external = parseExternal(externalUri); - if (external.isLeft()) { - return Optional.of(external.left().get()); + try { + URI externalUri = new URI(href); + final String scheme = externalUri.getScheme(); + // If the URI has no scheme specified, + // we have to check if it is a relative path, and if yes, try to + // interpret it relative to the moduleLoadPath property of the current + // XQuery context. + if (scheme == null && moduleLoadPath != null) { + final String path = externalUri.getSchemeSpecificPart(); + Path f = Paths.get(path); + if (!f.isAbsolute()) { + if (moduleLoadPath.startsWith(XmldbURI.XMLDB_URI_PREFIX)) { + final XmldbURI parentUri = XmldbURI.create(moduleLoadPath); + docUri = parentUri.append(path); + return Either.Right(Tuple(params, docUri, MaybeLockedDocument.of(serializer.broker.getXMLResource(docUri, Lock.LockMode.READ_LOCK)))); } else { - memtreeDoc = external.right().get(); + f = Paths.get(moduleLoadPath, path); + externalUri = f.toUri(); } } - } catch (final PermissionDeniedException e) { - return Optional.of(new ResourceError("Permission denied on XInclude'd resource", e)); - } catch (final ParserConfigurationException | URISyntaxException e) { - throw new SAXException("XInclude: failed to parse document at URI: " + href + ": " + e.getMessage(), e); } - } - /* if document has not been found and xpointer is - * null, throw an exception. If xpointer != null - * we retry below and interpret docName as - * a collection. - */ - if (doc == null && memtreeDoc == null && xpointer == null) { - return Optional.of(new ResourceError("document " + docUri + " not found")); - } + final Either external = parseExternal(externalUri); + final Tuple2, XmldbURI> paramsAndDocUri = Tuple(params, docUri); + return external.map(doc -> paramsAndDocUri.before(MaybeLockedDocument.of(doc))); - if (xpointer == null && !xqueryDoc) { - // no xpointer found - just serialize the doc - if (memtreeDoc == null) { - serializer.serializeToReceiver(doc, false); - } else { - serializer.serializeToReceiver(memtreeDoc, false); - } - } else { - // process the xpointer or the stored XQuery - try { - Source source = null; - if (xpointer == null) { - source = new DBSource(serializer.broker, (BinaryDocument) doc, true); - } else { - xpointer = checkNamespaces(xpointer); - source = new StringSource(xpointer); - } - - final String xpointerCopy = xpointer; - final DocumentImpl docCopy = doc; - final XmldbURI docUriCopy = docUri; - final ConsumerE setupXqueryContextPreCompilation = xqueryContext -> { - if (namespaces != null) { - xqueryContext.declareNamespaces(namespaces); - } - xqueryContext.declareNamespace("xinclude", Namespaces.XINCLUDE_NS); - - // Setup the HTTP context if known - if (serializer.httpContext != null) { - xqueryContext.setHttpContext(serializer.httpContext); - } - - if (xpointerCopy != null) { - if (docCopy != null) { - xqueryContext.setStaticallyKnownDocuments(new XmldbURI[]{ docCopy.getURI() }); - } else if (docUriCopy != null) { - xqueryContext.setStaticallyKnownDocuments(new XmldbURI[]{ docUriCopy }); - } - } - }; - - final ConsumerE setupXqueryContextPreExecution = xqueryContext -> { - //TODO: change these to putting the XmldbURI in, but we need to warn users! - if (document != null) { - xqueryContext.declareVariable("xinclude:current-doc", true, document.getFileURI().toString()); - xqueryContext.declareVariable("xinclude:current-collection", true, document.getCollection().getURI().toString()); - } + } catch (final PermissionDeniedException e) { + return Either.Left(new ResourceError("Permission denied on XInclude'd resource", e)); + } catch (final ParserConfigurationException | URISyntaxException e) { + throw new SAXException("XInclude: failed to parse document at URI: " + href + ": " + e.getMessage(), e); + } + } - // pass parameters as variables - if (params != null) { - for (final Map.Entry entry : params.entrySet()) { - xqueryContext.declareVariable(entry.getKey(), true, entry.getValue()); - } - } - }; + private static class MaybeLockedDocument implements AutoCloseable { + private final Either document; - Sequence contextSeq = null; - if (memtreeDoc != null) { - contextSeq = memtreeDoc; - } + private MaybeLockedDocument(final org.exist.dom.memtree.DocumentImpl memtreeDoc) { + this.document = Either.Left(memtreeDoc); + } - try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(serializer.broker, source, xpointer != null, true, contextSeq, null, setupXqueryContextPreCompilation, setupXqueryContextPreExecution, null)) { + private MaybeLockedDocument(final org.exist.dom.persistent.LockedDocument lockedDocument) { + this.document = Either.Right(lockedDocument); + } - final Sequence seq = queryResult.result; + public static @Nullable MaybeLockedDocument of(final org.exist.dom.memtree.DocumentImpl memtreeDoc) { + if (memtreeDoc == null) { + return null; + } + return new MaybeLockedDocument(memtreeDoc); + } - if (Type.subTypeOf(seq.getItemType(), Type.NODE)) { - if (LOG.isDebugEnabled()) { - LOG.debug("XPointer found: {}", seq.getItemCount()); - } + public static @Nullable MaybeLockedDocument of(final org.exist.dom.persistent.LockedDocument lockedDocument) { + if (lockedDocument == null) { + return null; + } + return new MaybeLockedDocument(lockedDocument); + } - NodeValue node; - for (final SequenceIterator i = seq.iterate(); i.hasNext(); ) { - node = (NodeValue) i.nextItem(); - serializer.serializeToReceiver(node, false); - } - } else { - String val; - for (int i = 0; i < seq.getItemCount(); i++) { - val = seq.itemAt(i).getStringValue(); - characters(val); - } - } - } + public Document getDocument() { + return document.fold(memtreeDoc -> memtreeDoc, LockedDocument::getDocument); + } - } catch (final XPathException | IOException | PermissionDeniedException e) { - LOG.warn("XPointer error: {}", e.getMessage(), e); - throw new SAXException("Error while processing XInclude expression: " + e.getMessage(), e); + @Override + public void close() { + if (this.document.isRight()) { + final LockedDocument lockedDocument = this.document.right().get(); + lockedDocument.close(); } } - // restore settings - document = prevDoc; - serializer.createContainerElements = createContainerElements; - - return Optional.empty(); } private Either parseExternal(final URI externalUri) throws ParserConfigurationException, SAXException { @@ -529,7 +595,7 @@ private Either parseExternal( // we use eXist's in-memory DOM implementation final XMLReaderPool parserPool = serializer.broker.getBrokerPool().getParserPool(); - XMLReader reader = null; + @Nullable XMLReader reader = null; try (final InputStream is = con.getInputStream()) { final InputSource src = new InputSource(is); @@ -564,7 +630,7 @@ public void startPrefixMapping(final String prefix, final String uri) throws SAX * Process xmlns() schema. We process these here, because namespace mappings should * already been known when parsing the xpointer() expression. */ - private String checkNamespaces(String xpointer) throws XPathException { + private String checkNamespaces(String xpointer) throws IllegalArgumentException { int p0; while ((p0 = xpointer.indexOf("xmlns(")) != Constants.STRING_NOT_FOUND) { if (p0 < 0) { @@ -572,13 +638,13 @@ private String checkNamespaces(String xpointer) throws XPathException { } final int p1 = xpointer.indexOf(')', p0 + 6); if (p1 < 0) { - throw new XPathException((Expression) null, "expected ) for xmlns()"); + throw new IllegalArgumentException("expected ) for xmlns()"); } final String mapping = xpointer.substring(p0 + 6, p1); xpointer = xpointer.substring(0, p0) + xpointer.substring(p1 + 1); final StringTokenizer tok = new StringTokenizer(mapping, "= \t\n"); if (tok.countTokens() < 2) { - throw new XPathException((Expression) null, "expected prefix=namespace mapping in " + mapping); + throw new IllegalArgumentException("expected prefix=namespace mapping in " + mapping); } final String prefix = tok.nextToken(); final String namespaceURI = tok.nextToken(); diff --git a/exist-core/src/main/java/org/exist/test/ExistEmbeddedServer.java b/exist-core/src/main/java/org/exist/test/ExistEmbeddedServer.java index bdfae3061a..78c50e3fea 100644 --- a/exist-core/src/main/java/org/exist/test/ExistEmbeddedServer.java +++ b/exist-core/src/main/java/org/exist/test/ExistEmbeddedServer.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -24,9 +48,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.exist.EXistException; +import org.exist.repo.AutoDeploymentTrigger; import org.exist.start.Classpath; import org.exist.start.EXistClassLoader; import org.exist.storage.BrokerPool; +import org.exist.storage.BrokerPoolConstants; import org.exist.storage.journal.Journal; import org.exist.util.Configuration; import org.exist.util.ConfigurationHelper; @@ -39,12 +65,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; -import static org.exist.repo.AutoDeploymentTrigger.AUTODEPLOY_PROPERTY; - /** * Exist embedded Server Rule for JUnit. */ @@ -54,14 +79,14 @@ public class ExistEmbeddedServer extends ExternalResource { public static final String USE_TEMPORARY_STORAGE_PROPERTY = "exist.use-temporary-storage"; - private final Optional instanceName; - private final Optional configFile; - private final Optional configProperties; + private final String instanceName; + private final Path home; + private @Nullable final Path configFile; + private final Properties configProperties; private final boolean useTemporaryStorage; private final boolean disableAutoDeploy; - private Optional temporaryStorage = Optional.empty(); + private @Nullable Path temporaryStorage = null; - private String prevAutoDeploy = "off"; private BrokerPool pool = null; public ExistEmbeddedServer() { @@ -97,9 +122,10 @@ public ExistEmbeddedServer(final String instanceName, final Path configFile, fin } public ExistEmbeddedServer(@Nullable final String instanceName, @Nullable final Path configFile, @Nullable final Properties configProperties, final boolean disableAutoDeploy, final boolean useTemporaryStorage) { - this.instanceName = Optional.ofNullable(instanceName); - this.configFile = Optional.ofNullable(configFile); - this.configProperties = Optional.ofNullable(configProperties); + this.instanceName = instanceName != null ? instanceName : BrokerPool.DEFAULT_INSTANCE_NAME; + this.home = Paths.get(System.getProperty("exist.home", System.getProperty("user.dir"))); + this.configFile = configFile != null ? configFile : ConfigurationHelper.lookup("conf.xml", Optional.of(home)); + this.configProperties = configProperties != null ? configProperties : new Properties(); this.disableAutoDeploy = disableAutoDeploy; this.useTemporaryStorage = useTemporaryStorage; @@ -118,43 +144,41 @@ protected void before() throws Throwable { public void startDb() throws DatabaseConfigurationException, EXistException, IOException { if(pool == null) { - if(disableAutoDeploy) { - this.prevAutoDeploy = System.getProperty(AUTODEPLOY_PROPERTY, "off"); - System.setProperty(AUTODEPLOY_PROPERTY, "off"); - } - - final String name = instanceName.orElse(BrokerPool.DEFAULT_INSTANCE_NAME); - - final Optional home = Optional.ofNullable(System.getProperty("exist.home", System.getProperty("user.dir"))).map(Paths::get); - final Path confFile = configFile.orElseGet(() -> ConfigurationHelper.lookup("conf.xml", home)); - final Configuration config; - if(confFile.isAbsolute() && Files.exists(confFile)) { - //TODO(AR) is this correct? - config = new Configuration(confFile.toAbsolutePath().toString()); + if(configFile.isAbsolute() && Files.exists(configFile)) { + config = new Configuration(configFile.toAbsolutePath().toString()); } else { - config = new Configuration(FileUtils.fileName(confFile), home); + config = new Configuration(FileUtils.fileName(configFile), Optional.of(home)); } - // override any specified config properties - configProperties.ifPresent(properties -> { - for (final Map.Entry configProperty : properties.entrySet()) { - config.setProperty(configProperty.getKey().toString(), configProperty.getValue()); - } - }); - final boolean propUseTemporaryStorage = Boolean.parseBoolean(System.getProperty(USE_TEMPORARY_STORAGE_PROPERTY, "false")); if (useTemporaryStorage || propUseTemporaryStorage) { - if (!temporaryStorage.isPresent()) { - this.temporaryStorage = Optional.of(Files.createTempDirectory("org.exist.test.ExistEmbeddedServer")); + if (temporaryStorage == null) { + this.temporaryStorage = Files.createTempDirectory("org.exist.test.ExistEmbeddedServer"); } - config.setProperty(BrokerPool.PROPERTY_DATA_DIR, temporaryStorage.get()); - config.setProperty(Journal.PROPERTY_RECOVERY_JOURNAL_DIR, temporaryStorage.get()); - LOG.info("Using temporary storage location: {}", temporaryStorage.get().toAbsolutePath().toString()); + configProperties.put(BrokerPool.PROPERTY_DATA_DIR, temporaryStorage); + configProperties.put(Journal.PROPERTY_RECOVERY_JOURNAL_DIR, temporaryStorage); + LOG.info("Using temporary storage location: {}", temporaryStorage.toAbsolutePath().toString()); } - BrokerPool.configure(name, 1, 5, config, Optional.empty()); - this.pool = BrokerPool.getInstance(name); + // override any specified config properties + for (final Map.Entry configProperty : configProperties.entrySet()) { + config.setProperty(configProperty.getKey().toString(), configProperty.getValue()); + } + + if (disableAutoDeploy) { + // remove auto deploy from config if present + final List configuredStartupTriggers = (List) config.getProperty(BrokerPoolConstants.PROPERTY_STARTUP_TRIGGERS); + for (final Configuration.StartupTriggerConfig configuredStartupTrigger : configuredStartupTriggers) { + if (AutoDeploymentTrigger.class.getName().equals(configuredStartupTrigger.getClazz())) { + configuredStartupTriggers.remove(configuredStartupTrigger); + break; + } + } + } + + BrokerPool.configure(instanceName, 1, 5, config, Optional.empty()); + this.pool = BrokerPool.getInstance(instanceName); } else { throw new IllegalStateException("ExistEmbeddedServer already running"); } @@ -164,7 +188,7 @@ public BrokerPool getBrokerPool() { return pool; } - public Optional getTemporaryStorage() { + public @Nullable Path getTemporaryStorage() { return temporaryStorage; } @@ -200,14 +224,9 @@ public void stopDb(final boolean clearTemporaryStorage) { pool = null; final boolean propUseTemporaryStorage = Boolean.parseBoolean(System.getProperty(USE_TEMPORARY_STORAGE_PROPERTY, "false")); - if((useTemporaryStorage || propUseTemporaryStorage) && temporaryStorage.isPresent() && clearTemporaryStorage) { - FileUtils.deleteQuietly(temporaryStorage.get()); - temporaryStorage = Optional.empty(); - } - - if(disableAutoDeploy) { - //set the autodeploy trigger enablement back to how it was before this test class - System.setProperty(AUTODEPLOY_PROPERTY, this.prevAutoDeploy); + if((useTemporaryStorage || propUseTemporaryStorage) && temporaryStorage != null && clearTemporaryStorage) { + FileUtils.deleteQuietly(temporaryStorage); + temporaryStorage = null; } } else { diff --git a/exist-core/src/main/java/org/exist/test/ExistWebServer.java b/exist-core/src/main/java/org/exist/test/ExistWebServer.java index 00bfcf9d7a..ece272f4ea 100644 --- a/exist-core/src/main/java/org/exist/test/ExistWebServer.java +++ b/exist-core/src/main/java/org/exist/test/ExistWebServer.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -28,14 +52,17 @@ import org.exist.collections.triggers.TriggerException; import org.exist.jetty.JettyStart; import org.exist.security.PermissionDeniedException; +import org.exist.storage.BrokerPool; +import org.exist.storage.journal.Journal; import org.exist.util.FileUtils; import org.exist.util.LockException; import org.junit.rules.ExternalResource; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Optional; +import java.util.Properties; import static org.exist.util.IPUtil.nextFreePort; import static org.junit.Assert.fail; @@ -50,11 +77,8 @@ public class ExistWebServer extends ExternalResource { public static final String USE_TEMPORARY_STORAGE_PROPERTY = "exist.use-temporary-storage"; - private static final String CONFIG_PROP_FILES = "org.exist.db-connection.files"; - private static final String CONFIG_PROP_JOURNAL_DIR = "org.exist.db-connection.recovery.journal-dir"; - private static final String PROP_JETTY_PORT = "jetty.port"; - private static final String PROP_JETTY_SECURE_PORT = "jetty.secure.port"; + private static final String PROP_JETTY_SECURE_PORT = "jetty.httpConfig.securePort"; private static final String PROP_JETTY_SSL_PORT = "jetty.ssl.port"; private static final int MIN_RANDOM_PORT = 49152; @@ -62,13 +86,13 @@ public class ExistWebServer extends ExternalResource { private static final int MAX_RANDOM_PORT_ATTEMPTS = 10; private JettyStart server = null; - private String prevAutoDeploy = "off"; private final boolean useRandomPort; private final boolean cleanupDbOnShutdown; + private final Properties configProperties; private final boolean disableAutoDeploy; private final boolean useTemporaryStorage; - private Optional temporaryStorage = Optional.empty(); + private @Nullable Path temporaryStorage = null; private final boolean jettyStandaloneMode; public ExistWebServer() { @@ -92,8 +116,13 @@ public ExistWebServer(final boolean useRandomPort, final boolean cleanupDbOnShut } public ExistWebServer(final boolean useRandomPort, final boolean cleanupDbOnShutdown, final boolean disableAutoDeploy, final boolean useTemporaryStorage, final boolean jettyStandaloneMode) { + this(useRandomPort, cleanupDbOnShutdown, null, disableAutoDeploy, useTemporaryStorage, jettyStandaloneMode); + } + + public ExistWebServer(final boolean useRandomPort, final boolean cleanupDbOnShutdown, @Nullable final Properties configProperties, final boolean disableAutoDeploy, final boolean useTemporaryStorage, final boolean jettyStandaloneMode) { this.useRandomPort = useRandomPort; this.cleanupDbOnShutdown = cleanupDbOnShutdown; + this.configProperties = configProperties != null ? configProperties : new Properties(); this.disableAutoDeploy = disableAutoDeploy; this.useTemporaryStorage = useTemporaryStorage; this.jettyStandaloneMode = jettyStandaloneMode; @@ -109,37 +138,36 @@ public final int getPort() { @Override protected void before() throws Throwable { - if(disableAutoDeploy) { - this.prevAutoDeploy = System.getProperty(AUTODEPLOY_PROPERTY, "off"); - System.setProperty(AUTODEPLOY_PROPERTY, "off"); - } - if (server == null) { final boolean propUseTemporaryStorage = Boolean.parseBoolean(System.getProperty(USE_TEMPORARY_STORAGE_PROPERTY, "false")); - if(useTemporaryStorage || propUseTemporaryStorage) { - this.temporaryStorage = Optional.of(Files.createTempDirectory("org.exist.test.ExistWebServer")); - final String absTemporaryStorage = temporaryStorage.get().toAbsolutePath().toString(); - System.setProperty(CONFIG_PROP_FILES, absTemporaryStorage); - System.setProperty(CONFIG_PROP_JOURNAL_DIR, absTemporaryStorage); - LOG.info("Using temporary storage location: {}", absTemporaryStorage); + if (useTemporaryStorage || propUseTemporaryStorage) { + if (temporaryStorage == null) { + this.temporaryStorage = Files.createTempDirectory("org.exist.test.ExistWebServer"); + } + configProperties.put(BrokerPool.PROPERTY_DATA_DIR, temporaryStorage); + configProperties.put(Journal.PROPERTY_RECOVERY_JOURNAL_DIR, temporaryStorage); + LOG.info("Using temporary storage location: {}", temporaryStorage.toAbsolutePath().toString()); } + final Properties jettyConfigProperties = new Properties(); if(useRandomPort) { - synchronized(ExistWebServer.class) { - System.setProperty(PROP_JETTY_PORT, Integer.toString(nextFreePort(MIN_RANDOM_PORT, MAX_RANDOM_PORT, MAX_RANDOM_PORT_ATTEMPTS))); - System.setProperty(PROP_JETTY_SECURE_PORT, Integer.toString(nextFreePort(MIN_RANDOM_PORT, MAX_RANDOM_PORT, MAX_RANDOM_PORT_ATTEMPTS))); - System.setProperty(PROP_JETTY_SSL_PORT, Integer.toString(nextFreePort(MIN_RANDOM_PORT, MAX_RANDOM_PORT, MAX_RANDOM_PORT_ATTEMPTS))); + jettyConfigProperties.setProperty(PROP_JETTY_PORT, Integer.toString(nextFreePort(MIN_RANDOM_PORT, MAX_RANDOM_PORT, MAX_RANDOM_PORT_ATTEMPTS))); + jettyConfigProperties.setProperty(PROP_JETTY_SECURE_PORT, Integer.toString(nextFreePort(MIN_RANDOM_PORT, MAX_RANDOM_PORT, MAX_RANDOM_PORT_ATTEMPTS))); + jettyConfigProperties.setProperty(PROP_JETTY_SSL_PORT, Integer.toString(nextFreePort(MIN_RANDOM_PORT, MAX_RANDOM_PORT, MAX_RANDOM_PORT_ATTEMPTS))); + } - server = new JettyStart(); - server.run(jettyStandaloneMode); - } - } else { - server = new JettyStart(); - server.run(); + if (disableAutoDeploy) { + // NOTE(AR) will be processed in JettyStart + configProperties.setProperty(AUTODEPLOY_PROPERTY, "off"); } + + server = new JettyStart(configProperties, jettyConfigProperties); + server.run(jettyStandaloneMode); + } else { throw new IllegalStateException("ExistWebServer already running"); } + super.before(); } @@ -170,29 +198,14 @@ protected void after() { server = null; final boolean propUseTemporaryStorage = Boolean.parseBoolean(System.getProperty(USE_TEMPORARY_STORAGE_PROPERTY, "false")); - if((useTemporaryStorage || propUseTemporaryStorage) && temporaryStorage.isPresent()) { - FileUtils.deleteQuietly(temporaryStorage.get()); - temporaryStorage = Optional.empty(); - System.clearProperty(CONFIG_PROP_JOURNAL_DIR); - System.clearProperty(CONFIG_PROP_FILES); - } - - if(useRandomPort) { - synchronized (ExistWebServer.class) { - System.clearProperty(PROP_JETTY_SSL_PORT); - System.clearProperty(PROP_JETTY_SECURE_PORT); - System.clearProperty(PROP_JETTY_PORT); - } + if((useTemporaryStorage || propUseTemporaryStorage) && temporaryStorage != null) { + FileUtils.deleteQuietly(temporaryStorage); + temporaryStorage = null; } } else { throw new IllegalStateException("ExistWebServer already stopped"); } - if(disableAutoDeploy) { - //set the autodeploy trigger enablement back to how it was before this test class - System.setProperty(AUTODEPLOY_PROPERTY, this.prevAutoDeploy); - } - super.after(); } } diff --git a/exist-core/src/main/java/org/exist/util/IPUtil.java b/exist-core/src/main/java/org/exist/util/IPUtil.java index 3b1ce398e5..ead0636a25 100644 --- a/exist-core/src/main/java/org/exist/util/IPUtil.java +++ b/exist-core/src/main/java/org/exist/util/IPUtil.java @@ -46,7 +46,7 @@ public class IPUtil { @GuardedBy("class") - private static final Random random = new Random(); + private static final Random RANDOM = new Random(); /** * Attempts to get the next random free IP port in the range {@code from} and {@code to}. @@ -72,7 +72,7 @@ public static int nextFreePort(final int from, final int to, final int maxAttemp } private synchronized static int random(final int min, final int max) { - return random.nextInt((max - min) + 1) + min; + return RANDOM.nextInt((max - min) + 1) + min; } private static boolean isLocalPortFree(final int port) { diff --git a/exist-core/src/main/java/org/exist/xmldb/LocalCollection.java b/exist-core/src/main/java/org/exist/xmldb/LocalCollection.java index c7484b4df5..ada1aa0166 100644 --- a/exist-core/src/main/java/org/exist/xmldb/LocalCollection.java +++ b/exist-core/src/main/java/org/exist/xmldb/LocalCollection.java @@ -323,7 +323,7 @@ public org.xmldb.api.base.Collection getParentCollection() throws XMLDBException }); } - public String getPath() throws XMLDBException { + String getPath() { return path.toString(); } @@ -765,13 +765,8 @@ public XmldbURI getURI() { //No port ;-) //No context ;-) //accessor.append(getContext()); - try { - //TODO : cache it when constructed - return XmldbURI.create(accessor.toString(), getPath()); - } catch(final XMLDBException e) { - //TODO : should never happen - return null; - } + + return XmldbURI.create(accessor.toString(), getPath()); } /** diff --git a/exist-core/src/main/java/org/exist/xmldb/LocalCompiledExpression.java b/exist-core/src/main/java/org/exist/xmldb/LocalCompiledExpression.java new file mode 100644 index 0000000000..2c37a16b5a --- /dev/null +++ b/exist-core/src/main/java/org/exist/xmldb/LocalCompiledExpression.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2014, Evolved Binary Ltd + * + * This file was originally ported from FusionDB to Elemental by + * Evolved Binary, for the benefit of the Elemental Open Source community. + * Only the ported code as it appears in this file, at the time that + * it was contributed to Elemental, was re-licensed under The GNU + * Lesser General Public License v2.1 only for use in Elemental. + * + * This license grant applies only to a snapshot of the code as it + * appeared when ported, it does not offer or infer any rights to either + * updates of this source code or access to the original source code. + * + * The GNU Lesser General Public License v2.1 only license follows. + * + * ===================================================================== + * + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xmldb; + +import org.exist.xquery.XQueryUtil; +import org.xmldb.api.base.CompiledExpression; + +/** + * XML:DB Local Compiled Expression wrapper for a compiled XQuery. + * + * @author Adam Retter + */ +public class LocalCompiledExpression implements CompiledExpression, AutoCloseable { + + private final XQueryUtil.CompilationResult compilationResult; + + public LocalCompiledExpression(final XQueryUtil.CompilationResult compilationResult) { + this.compilationResult = compilationResult; + } + + /** + * Get the underlying compilation result. + * + * @return the underlying compilation result. + */ + public XQueryUtil.CompilationResult getCompilationResult() { + return compilationResult; + } + + @Override + public void reset() { + compilationResult.reset(); + } + + @Override + public void close() { + compilationResult.close(); + } +} diff --git a/exist-core/src/main/java/org/exist/xmldb/LocalXPathQueryService.java b/exist-core/src/main/java/org/exist/xmldb/LocalXPathQueryService.java index 702cc6b9eb..bd783408a9 100644 --- a/exist-core/src/main/java/org/exist/xmldb/LocalXPathQueryService.java +++ b/exist-core/src/main/java/org/exist/xmldb/LocalXPathQueryService.java @@ -51,11 +51,14 @@ import org.exist.EXistException; import org.exist.debuggee.Debuggee; import org.exist.dom.QName; +import org.exist.dom.persistent.LockedDocument; import org.exist.security.PermissionDeniedException; import org.exist.security.Subject; import org.exist.source.DBSource; +import org.exist.source.DbStoreSource; import org.exist.source.FileSource; import org.exist.source.Source; +import org.exist.source.StringSource; import org.exist.storage.BrokerPool; import org.exist.storage.DBBroker; import org.exist.storage.lock.Lock.LockMode; @@ -64,9 +67,7 @@ import org.exist.storage.txn.Txn; import org.exist.util.LockException; import org.exist.xmldb.function.LocalXmldbFunction; -import org.exist.xquery.CompiledXQuery; import org.exist.xquery.XPathException; -import org.exist.xquery.XQuery; import org.exist.xquery.XQueryContext; import org.exist.xquery.XQueryUtil; import org.exist.xquery.value.AnyURIValue; @@ -79,6 +80,7 @@ import org.xmldb.api.base.ErrorCodes; import org.xmldb.api.modules.XMLResource; +import java.io.IOException; import java.io.Writer; import java.util.*; @@ -89,7 +91,6 @@ import org.exist.dom.persistent.MutableDocumentSet; import org.exist.dom.persistent.NodeProxy; import org.exist.dom.persistent.NodeSet; -import org.exist.security.Permission; import com.evolvedbinary.j8fu.Either; import javax.annotation.Nullable; @@ -181,7 +182,7 @@ public EXistResourceSet query(final XMLResource res, final String query, final S } private EXistResourceSet doQuery(final DBBroker broker, final Txn transaction, final String query, final XmldbURI[] docs, final Sequence contextSet, final String sortExpr) throws XMLDBException { - final Either maybeExpr = compileAndCheck(broker, transaction, query); + final Either maybeExpr = compileAndCheck(broker, transaction, query); if(maybeExpr.isLeft()) { final XPathException e = maybeExpr.left().get(); throw new XMLDBException(ErrorCodes.VENDOR_ERROR, e.getMessage(), e); @@ -214,34 +215,27 @@ public EXistResourceSet execute(final XMLResource res, final CompiledExpression } private EXistResourceSet execute(final DBBroker broker, final Txn transaction, XmldbURI[] docs, final Sequence contextSet, final CompiledExpression expression, final String sortExpr) throws XMLDBException { - final CompiledXQuery expr = (CompiledXQuery) expression; - final XQueryContext context = expr.getContext(); + if (!(expression instanceof LocalCompiledExpression)) { + throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "LocalXPathQueryService#execute requires a LocalCompiledExpression"); + } + final LocalCompiledExpression localCompiledExpression = (LocalCompiledExpression) expression; + final XQueryUtil.CompilationResult compilationResult = localCompiledExpression.getCompilationResult(); - boolean queryResultOwnershipTransferred = false; - @Nullable XQueryUtil.QueryResult queryResult = null; - try { - context.setStaticallyKnownDocuments(docs); + final ConsumerE preExecutionContext = xqueryContext -> { + xqueryContext.setStaticallyKnownDocuments(docs); if (lockedDocuments != null) { - context.setProtectedDocs(lockedDocuments); - } - setupContext(null, context); - declareVariables(context); - - final XQuery xquery = brokerPool.getXQueryService(); - - final long executionStarted = System.currentTimeMillis(); - @Nullable final Sequence result = xquery.execute(broker, expr, null, contextSet, properties, true); - final long executionFinished = System.currentTimeMillis(); - - if (LOG.isTraceEnabled()) { - LOG.trace("Query took {} ms.", executionFinished - executionStarted); - } - - if (result == null) { - return null; + xqueryContext.setProtectedDocs(lockedDocuments); } + setupContext(compilationResult.source, xqueryContext); + declareVariables(xqueryContext); + }; - queryResult = new XQueryUtil.QueryResult(expr.getSource(), null, expr, context, -1, executionFinished - executionStarted, result); + boolean queryResultOwnershipTransferred = false; + @Nullable XQueryUtil.QueryResult queryResult = null; + try { + // TODO(AR) does reset context need to be called on the result? it was previously (note the 'true' param) -> xquery.execute(broker, expr, null, contextSet, properties, true); + // NOTE(AR) queryResult takes ownership of compilationResult + queryResult = XQueryUtil.execute(broker, compilationResult, contextSet, properties, preExecutionContext, null); final Properties resourceSetProperties = new Properties(properties); resourceSetProperties.setProperty(EXistOutputKeys.XDM_SERIALIZATION, "yes"); @@ -257,11 +251,11 @@ private EXistResourceSet execute(final DBBroker broker, final Txn transaction, X } finally { if (!queryResultOwnershipTransferred) { - // NOTE(AR) we are returning null or an exception was raised, and so we still own the queryResult and we must therefore close it + // NOTE(AR) we are returning null or an exception was raised, and so we still own the queryResult or compilationResult and we must therefore close it if (queryResult != null) { queryResult.close(); } else { - context.runCleanupTasks(); + compilationResult.close(); } } } @@ -274,72 +268,90 @@ public EXistResourceSet execute(final Source source) throws XMLDBException { @Override public EXistResourceSet executeStoredQuery(final String uri) throws XMLDBException { - return execute((broker, transaction) -> { - final DocumentImpl resource = broker.getResource(new XmldbURI(uri), Permission.READ | Permission.EXECUTE); - if (resource == null) { - throw new XMLDBException(ErrorCodes.INVALID_URI, "No stored XQuery exists at: " + uri); + return withDb((broker, transaction) -> { + + final Either maybeExpr; + + try (final LockedDocument lockedResource = broker.getXMLResource(new XmldbURI(uri), LockMode.READ_LOCK)) { + if (lockedResource == null) { + throw new XMLDBException(ErrorCodes.INVALID_URI, "No stored XQuery exists at: " + uri); + } + final DocumentImpl resource = lockedResource.getDocument(); + if (!(resource instanceof BinaryDocument)) { + throw new XMLDBException(ErrorCodes.INVALID_URI, "Document is not an XQuery: " + uri); + } + + final Source dbSource = new DBSource(broker.getBrokerPool(), (BinaryDocument) resource, false); + maybeExpr = compileAndCheck(broker, transaction, dbSource); + } + + // NOTE(AR) query is now compiled so we can release the LockedDocument eagerly above + if (maybeExpr.isLeft()) { + final XPathException e = maybeExpr.left().get(); + throw new XMLDBException(ErrorCodes.VENDOR_ERROR, e.getMessage(), e); + } else { + return execute(broker, transaction, null, null, maybeExpr.right().get(), null); } - return new DBSource(broker, (BinaryDocument) resource, false); }); } private EXistResourceSet execute(final LocalXmldbFunction sourceOp) throws XMLDBException { return withDb((broker, transaction) -> { - final Source source = sourceOp.apply(broker, transaction); - final XmldbURI[] docs = new XmldbURI[]{ XmldbURI.create(collection.getName(broker, transaction)) }; + return execute(broker, transaction, source); + }); + } - final ConsumerE setupXqueryContextPreCompilation = xqueryContext -> { - xqueryContext.setStaticallyKnownDocuments(docs); - try { - setupContext(source, xqueryContext); - } catch (final XMLDBException e) { - throw new XPathException(e); - } - }; + private EXistResourceSet execute(final DBBroker broker, final Txn transaction, final Source source) throws XMLDBException, XPathException, PermissionDeniedException, IOException { - final ConsumerE setupXqueryContextPreExecution = this::declareVariables; + final XmldbURI[] docs = new XmldbURI[]{ XmldbURI.create(collection.getName(broker, transaction)) }; - boolean queryResultOwnershipTransferred = false; - @Nullable XQueryUtil.QueryResult queryResult = null; - try { - queryResult = XQueryUtil.query(broker, source, true, null, null, setupXqueryContextPreCompilation, setupXqueryContextPreExecution, null); + final ConsumerE setupXqueryContextPreCompilation = xqueryContext -> { + xqueryContext.setStaticallyKnownDocuments(docs); + setupContext(source, xqueryContext); + }; - if (LOG.isTraceEnabled()) { - LOG.trace("Query took {} ms.", queryResult.executionTime); - } + final ConsumerE setupXqueryContextPreExecution = this::declareVariables; - if (queryResult.result == null) { - return null; - } + boolean queryResultOwnershipTransferred = false; + @Nullable XQueryUtil.QueryResult queryResult = null; + try { + queryResult = XQueryUtil.query(broker, source, true, null, null, setupXqueryContextPreCompilation, setupXqueryContextPreExecution, null); - final Properties resourceSetProperties = new Properties(properties); - resourceSetProperties.setProperty(EXistOutputKeys.XDM_SERIALIZATION, "yes"); + if (LOG.isTraceEnabled()) { + LOG.trace("Query took {} ms.", queryResult.executionTime); + } - // NOTE(AR) LocalResourceSet takes ownership of QueryResult sequence! So it is responsible to call `QueryResult#close()` when it is finished with it - final EXistResourceSet resourceSet = new LocalResourceSet(user, brokerPool, collection, resourceSetProperties, queryResult, null); - queryResultOwnershipTransferred = true; - return resourceSet; + if (queryResult.result == null) { + return null; + } - } catch (final XPathException e) { - if (e.getCause() instanceof XMLDBException) { - throw (XMLDBException) e.getCause(); - } - throw e; + final Properties resourceSetProperties = new Properties(properties); + resourceSetProperties.setProperty(EXistOutputKeys.XDM_SERIALIZATION, "yes"); - } finally { - if (queryResult != null && !queryResultOwnershipTransferred) { - // NOTE(AR) we are returning null or an exception was raised, and so we still own the queryResult and we must therefore close it - queryResult.close(); - } + // NOTE(AR) LocalResourceSet takes ownership of QueryResult sequence! So it is responsible to call `QueryResult#close()` when it is finished with it + final EXistResourceSet resourceSet = new LocalResourceSet(user, brokerPool, collection, resourceSetProperties, queryResult, null); + queryResultOwnershipTransferred = true; + return resourceSet; + + } catch (final XPathException | PermissionDeniedException | IOException e) { + if (e.getCause() instanceof XMLDBException) { + throw (XMLDBException) e.getCause(); } - }); + throw e; + + } finally { + if (queryResult != null && !queryResultOwnershipTransferred) { + // NOTE(AR) we are returning null or an exception was raised, and so we still own the queryResult and we must therefore close it + queryResult.close(); + } + } } @Override public CompiledExpression compile(final String query) throws XMLDBException { return withDb((broker, transaction) -> { - final Either maybeExpr = compileAndCheck(broker, transaction, query); + final Either maybeExpr = compileAndCheck(broker, transaction, query); if(maybeExpr.isLeft()) { final XPathException e = maybeExpr.left().get(); throw new XMLDBException(ErrorCodes.VENDOR_ERROR, e.getMessage(), e); @@ -351,7 +363,7 @@ public CompiledExpression compile(final String query) throws XMLDBException { @Override public CompiledExpression compileAndCheck(final String query) throws XMLDBException, XPathException { - final Either result = withDb((broker, transaction) -> compileAndCheck(broker, transaction, query)); + final Either result = withDb((broker, transaction) -> compileAndCheck(broker, transaction, query)); if(result.isLeft()) { throw result.left().get(); } else { @@ -359,21 +371,25 @@ public CompiledExpression compileAndCheck(final String query) throws XMLDBExcept } } - private Either compileAndCheck(final DBBroker broker, final Txn transaction, final String query) throws XMLDBException { - final long start = System.currentTimeMillis(); - final XQuery xquery = broker.getBrokerPool().getXQueryService(); - final XQueryContext context = new XQueryContext(broker.getBrokerPool()); + private Either compileAndCheck(final DBBroker broker, final Txn transaction, final String query) throws XMLDBException { + final Source source = new StringSource(query); + return compileAndCheck(broker, transaction, source); + } + + private Either compileAndCheck(final DBBroker broker, final Txn transaction, final Source source) throws XMLDBException { + final ConsumerE preCompilationContext = xqueryContext -> setupContext(source, xqueryContext); try { - setupContext(null, context); - final CompiledExpression expr = xquery.compile(context, query); - if(LOG.isDebugEnabled()) { - LOG.debug("compilation took {}", System.currentTimeMillis() - start); + final XQueryUtil.CompilationResult compilationResult = XQueryUtil.compile(broker, source, false, true, preCompilationContext); + if (LOG.isDebugEnabled()) { + LOG.debug("Compilation took {}ms", compilationResult.compilationTime); } - return Either.Right(expr); + + final LocalCompiledExpression compiledExpression = new LocalCompiledExpression(compilationResult); + return Either.Right(compiledExpression); } catch (final PermissionDeniedException e) { throw new XMLDBException(ErrorCodes.PERMISSION_DENIED, e.getMessage(), e); - } catch(final IllegalArgumentException e) { + } catch(final IOException e) { throw new XMLDBException(ErrorCodes.VENDOR_ERROR, e.getMessage(), e); } catch(final XPathException e) { return Either.Left(e); @@ -392,19 +408,15 @@ public EXistResourceSet queryResource(final String resource, final String query) }); } - protected void setupContext(final Source source, final XQueryContext context) throws XMLDBException, XPathException { - try { - context.setBaseURI(new AnyURIValue(properties.getProperty("base-uri", collection.getPath()))); - } catch(final XPathException e) { - throw new XMLDBException(ErrorCodes.INVALID_URI,"Invalid base uri",e); - } + protected void setupContext(final Source source, final XQueryContext context) throws XPathException { + context.setBaseURI(new AnyURIValue(properties.getProperty("base-uri", collection.getPath()))); - if(moduleLoadPath != null) { + if (moduleLoadPath != null) { context.setModuleLoadPath(moduleLoadPath); } else if (source != null) { String modulePath = null; - if (source instanceof DBSource) { - modulePath = ((DBSource) source).getDocumentPath().removeLastSegment().toString(); + if (source instanceof DbStoreSource) { + modulePath = ((DbStoreSource) source).getDocumentPath().removeLastSegment().toString(); } else if (source instanceof FileSource) { modulePath = ((FileSource) source).getPath().getParent().toString(); } @@ -547,7 +559,10 @@ public void setModuleLoadPath(final String path) { @Override public void dump(final CompiledExpression expression, final Writer writer) throws XMLDBException { - final CompiledXQuery expr = (CompiledXQuery)expression; - expr.dump(writer); + if (!(expression instanceof LocalCompiledExpression)) { + throw new XMLDBException(ErrorCodes.VENDOR_ERROR, "LocalXPathQueryService#dump requires a LocalCompiledExpression"); + } + final LocalCompiledExpression localCompiledExpression = (LocalCompiledExpression) expression; + localCompiledExpression.getCompilationResult().dump(writer); } } diff --git a/exist-core/src/main/java/org/exist/xmlrpc/AbstractCachedResult.java b/exist-core/src/main/java/org/exist/xmlrpc/AbstractCachedResult.java index 86ad37b603..690149cf04 100644 --- a/exist-core/src/main/java/org/exist/xmlrpc/AbstractCachedResult.java +++ b/exist-core/src/main/java/org/exist/xmlrpc/AbstractCachedResult.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -37,7 +61,7 @@ public abstract class AbstractCachedResult implements Closeable { protected long queryTime = 0; protected long creationTimestamp = 0; protected long timestamp = 0; - private boolean closed; + private boolean closed = false; public AbstractCachedResult() { this(0); diff --git a/exist-core/src/main/java/org/exist/xmlrpc/CachedQueryResult.java b/exist-core/src/main/java/org/exist/xmlrpc/CachedQueryResult.java new file mode 100644 index 0000000000..07bfb40c29 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xmlrpc/CachedQueryResult.java @@ -0,0 +1,101 @@ +/* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xmlrpc; + +import org.exist.xquery.XQueryUtil; +import org.exist.xquery.value.Sequence; + +import java.util.Properties; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.annotation.Nullable; + +/** + * Simple container for the results of a query. Used to cache + * query results that may be retrieved later by the client. + * + * @author wolf + * @author jmfernandez + * @author Adam Retter + */ +public class CachedQueryResult extends AbstractCachedResult { + + private static final Logger LOG = LogManager.getLogger(CachedQueryResult.class); + private final XQueryUtil.QueryResult queryResult; + private @Nullable final Properties serialization; + + public CachedQueryResult(final XQueryUtil.QueryResult queryResult, @Nullable final Properties outputProperties) { + super(queryResult.totalTime()); + this.queryResult = queryResult; + this.serialization = outputProperties; + } + + /** + * Get the total time to produce the query result. + * This is compilation time (if any) plus the execution time. + * + * @return total time to produce the query result. + */ + public long totalTime() { + return queryResult.totalTime(); + } + + @Override + public Sequence getResult() { + return queryResult.result; + } + + public @Nullable Properties getSerialization() { + return serialization; + } + + @Override + protected void doClose() { + queryResult.close(); + } +} diff --git a/exist-core/src/main/java/org/exist/xmlrpc/QueryResult.java b/exist-core/src/main/java/org/exist/xmlrpc/QueryResult.java deleted file mode 100644 index 4465f00d83..0000000000 --- a/exist-core/src/main/java/org/exist/xmlrpc/QueryResult.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xmlrpc; - -import java.io.IOException; - -import org.exist.xquery.XPathException; -import org.exist.xquery.value.Sequence; - -import java.util.Properties; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.exist.xquery.value.BinaryValue; - -/** - * Simple container for the results of a query. Used to cache - * query results that may be retrieved later by the client. - * - * @author wolf - * @author jmfernandez - */ -public class QueryResult extends AbstractCachedResult { - - private final static Logger LOG = LogManager.getLogger(QueryResult.class); - protected Sequence result; - protected Properties serialization = null; - // set upon failure - protected XPathException exception = null; - - public QueryResult(final Sequence result, final Properties outputProperties) { - this(result, outputProperties, 0); - } - - public QueryResult(final Sequence result, final Properties outputProperties, final long queryTime) { - super(queryTime); - this.serialization = outputProperties; - this.result = result; - } - - public QueryResult(final XPathException e) { - exception = e; - } - - public boolean hasErrors() { - return exception != null; - } - - public XPathException getException() { - return exception; - } - - /** - * @return Returns the result. - */ - @Override - public Sequence getResult() { - return result; - } - - @Override - protected void doClose() { - if (result != null) { - - //cleanup any binary values - if (result instanceof BinaryValue) { - try { - ((BinaryValue) result).close(); - } catch (final IOException ioe) { - LOG.warn("Unable to cleanup BinaryValue: {}", result.hashCode(), ioe); - } - } - - result = null; - } - } -} diff --git a/exist-core/src/main/java/org/exist/xmlrpc/QueryResultCache.java b/exist-core/src/main/java/org/exist/xmlrpc/QueryResultCache.java index 5bdb34806a..a2c1942ced 100644 --- a/exist-core/src/main/java/org/exist/xmlrpc/QueryResultCache.java +++ b/exist-core/src/main/java/org/exist/xmlrpc/QueryResultCache.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,11 +47,12 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.RemovalListener; import net.jcip.annotations.ThreadSafe; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Date; +import javax.annotation.Nullable; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -43,19 +68,23 @@ public class QueryResultCache { private static final Logger LOG = LogManager.getLogger(QueryResultCache.class); private static final int TIMEOUT = 180_000; // ms (e.g. 2 minutes) + private static final RemovalListener REMOVAL_LISTENER = (cacheId, cachedResult, removalCause) -> { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing cached query result for cache: {}", cacheId); + } + + // NOTE(AR) make sure to release any resources still held by the cached result + cachedResult.close(); + }; + private final AtomicInteger cacheIdCounter = new AtomicInteger(); private final Cache cache; public QueryResultCache() { this.cache = Caffeine.newBuilder() .expireAfterAccess(TIMEOUT, TimeUnit.MILLISECONDS) - .removalListener((key, value, cause) -> { - final AbstractCachedResult qr = (AbstractCachedResult)value; - qr.close(); // must close associated resources - if(LOG.isDebugEnabled()) { - LOG.debug("Removing cached result set: {}", new Date(qr.getTimestamp())); - } - }).build(); + .removalListener(REMOVAL_LISTENER) + .build(); } public int add(final AbstractCachedResult qr) { @@ -64,24 +93,24 @@ public int add(final AbstractCachedResult qr) { return cacheId; } - public AbstractCachedResult get(final int cacheId) { + public @Nullable AbstractCachedResult get(final int cacheId) { if (cacheId < 0 || cacheId >= cacheIdCounter.get()) { return null; } return cache.getIfPresent(cacheId); } - public QueryResult getResult(final int cacheId) { + public @Nullable CachedQueryResult getResult(final int cacheId) { final AbstractCachedResult acr = get(cacheId); - return (acr != null && acr instanceof QueryResult) ? (QueryResult) acr : null; + return (acr != null && acr instanceof CachedQueryResult) ? (CachedQueryResult) acr : null; } - public SerializedResult getSerializedResult(final int cacheId) { + public @Nullable SerializedResult getSerializedResult(final int cacheId) { final AbstractCachedResult acr = get(cacheId); return (acr != null && acr instanceof SerializedResult) ? (SerializedResult) acr : null; } - public CachedContentFile getCachedContentFile(final int cacheId) { + public @Nullable CachedContentFile getCachedContentFile(final int cacheId) { final AbstractCachedResult acr = get(cacheId); return (acr != null && acr instanceof CachedContentFile) ? (CachedContentFile) acr : null; } diff --git a/exist-core/src/main/java/org/exist/xmlrpc/RpcConnection.java b/exist-core/src/main/java/org/exist/xmlrpc/RpcConnection.java index 0bb395837c..e3514ca8c4 100644 --- a/exist-core/src/main/java/org/exist/xmlrpc/RpcConnection.java +++ b/exist-core/src/main/java/org/exist/xmlrpc/RpcConnection.java @@ -45,6 +45,7 @@ */ package org.exist.xmlrpc; +import com.evolvedbinary.j8fu.function.BiConsumerE; import org.apache.commons.io.output.StringBuilderWriter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -106,7 +107,6 @@ import org.exist.validation.Validator; import org.exist.xmldb.XmldbURI; import org.exist.xmlrpc.function.XmlRpcCollectionFunction; -import org.exist.xmlrpc.function.XmlRpcCompiledXQueryFunction; import org.exist.xmlrpc.function.XmlRpcDocumentFunction; import org.exist.xmlrpc.function.XmlRpcFunction; import org.exist.xqj.Marshaller; @@ -123,7 +123,6 @@ import com.evolvedbinary.j8fu.function.ConsumerE; import com.evolvedbinary.j8fu.function.Function2E; -import com.evolvedbinary.j8fu.function.Function3E; import com.evolvedbinary.j8fu.function.SupplierE; import com.evolvedbinary.j8fu.tuple.Tuple2; @@ -157,7 +156,7 @@ * * @author Wolfgang Meier * Modified by {Marco.Tampucci, Massimo.Martinelli} @isti.cnr.it - * @author Adam Retter + * @author Adam Retter */ public class RpcConnection implements RpcAPI { @@ -264,54 +263,77 @@ private String createId(final XmldbURI collUri) throws EXistException, Permissio }); } - protected QueryResult doQuery(final DBBroker broker, final CompiledXQuery compiled, - final NodeSet contextSet, final Map parameters) throws XPathException, EXistException, PermissionDeniedException { - final XQuery xquery = broker.getBrokerPool().getXQueryService(); - final XQueryContext context = compiled.getContext(); - checkPragmas(context, parameters); - LockedDocumentMap lockedDocuments = null; + /** + * NOTE: Takes ownership of compiled CompilationResult. + */ + protected CachedQueryResult doQuery(final DBBroker broker, final XQueryUtil.CompilationResult compiled, final NodeSet contextSet, final Map parameters) throws XPathException, EXistException, PermissionDeniedException { + return doQuery(broker, compiled, contextSet, parameters, false); + } + + /** + * NOTE: Takes ownership of compiled CompilationResult. + */ + protected CachedQueryResult doQuery(final DBBroker broker, final XQueryUtil.CompilationResult compiled, final NodeSet contextSet, final Map parameters, final boolean extractSerializationOptions) throws XPathException, EXistException, PermissionDeniedException { + @Nullable LockedDocumentMap lockedDocuments = null; try { - // declare static variables - final Map variableDecls = (Map) parameters.get(RpcAPI.VARIABLES); - if (variableDecls != null) { - for (final Map.Entry entry : variableDecls.entrySet()) { - final String varNameStr = entry.getKey(); + try { + lockedDocuments = beginProtected(broker, parameters); + } catch (final EXistException | PermissionDeniedException e) { + compiled.close(); + throw e; + } + + @Nullable final LockedDocumentMap contextLockedDocuments = lockedDocuments; + final Holder xdmSerializationOptionsHolder = new Holder<>(null); + final ConsumerE preExecutionContext = xqueryContext -> { + checkPragmas(xqueryContext, parameters); + + // declare static variables + final Map variableDecls = (Map) parameters.get(RpcAPI.VARIABLES); + if (variableDecls != null) { + for (final Map.Entry entry : variableDecls.entrySet()) { + final String varNameStr = entry.getKey(); + + final QName varName; + try { + varName = QName.parse(xqueryContext, varNameStr); + } catch (final QName.IllegalQNameException e) { + throw new XPathException(org.exist.xquery.ErrorCodes.W3CErrorCode.XPST0081.getErrorCode(), "Error declaring variable, invalid qname: " + varNameStr + ". " + e.getMessage(), e); + } - final QName varName; - try { - varName = QName.parse(context, varNameStr); - } catch (final QName.IllegalQNameException e) { - throw new XPathException(org.exist.xquery.ErrorCodes.W3CErrorCode.XPST0081.getErrorCode(), "Error declaring variable, invalid qname: " + varNameStr + ". " + e.getMessage(), e); - } + if (!xqueryContext.isExternalVariableDeclared(varName)) { + throw new XPathException(org.exist.xquery.ErrorCodes.W3CErrorCode.XPDY0002.getErrorCode(), "External variable " + varName + " is not declared in the XQuery"); + } - if (!context.isExternalVariableDeclared(varName)) { - throw new XPathException(org.exist.xquery.ErrorCodes.W3CErrorCode.XPDY0002.getErrorCode(), "External variable " + varName + " is not declared in the XQuery"); + if (LOG.isDebugEnabled()) { + LOG.debug("declaring {} = {}", varName, entry.getValue()); + } + xqueryContext.declareVariable(varName, true, entry.getValue()); } + } - if (LOG.isDebugEnabled()) { - LOG.debug("declaring {} = {}", varName, entry.getValue()); - } - context.declareVariable(varName, true, entry.getValue()); + // let the context know about any lockedDocuments + if (contextLockedDocuments != null) { + xqueryContext.setProtectedDocs(contextLockedDocuments); } - } - final long start = System.currentTimeMillis(); - lockedDocuments = beginProtected(broker, parameters); - if (lockedDocuments != null) { - context.setProtectedDocs(lockedDocuments); - } - final Properties outputProperties = new Properties(); - final Sequence result = xquery.execute(broker, compiled, contextSet, outputProperties, true); + if (extractSerializationOptions) { + xdmSerializationOptionsHolder.value = getXdmSerializationOptions(xqueryContext); + } + }; + + final BiConsumerE postExecutionContext = (xqueryContext, queryResult) -> { + // pass last modified date to the HTTP response + HTTPUtils.addLastModifiedHeader(queryResult.result, xqueryContext); + }; + + final XQueryUtil.QueryResult queryResult = XQueryUtil.execute(broker, compiled, contextSet, null, preExecutionContext, postExecutionContext); if (LOG.isTraceEnabled()) { - LOG.trace("Query took {} ms.", System.currentTimeMillis() - start); + LOG.trace("Query took {} ms.", queryResult.totalTime()); } - // pass last modified date to the HTTP response - HTTPUtils.addLastModifiedHeader(result, context); + return new CachedQueryResult(queryResult, xdmSerializationOptionsHolder.value); - return new QueryResult(result, outputProperties); - } catch (final XPathException e) { - return new QueryResult(e); } finally { if (lockedDocuments != null) { lockedDocuments.unlock(); @@ -319,7 +341,7 @@ protected QueryResult doQuery(final DBBroker broker, final CompiledXQuery compil } } - protected LockedDocumentMap beginProtected(final DBBroker broker, final Map parameters) throws EXistException, PermissionDeniedException { + protected @Nullable LockedDocumentMap beginProtected(final DBBroker broker, final Map parameters) throws EXistException, PermissionDeniedException { final String protectColl = (String) parameters.get(RpcAPI.PROTECTED_MODE); if (protectColl == null) { return null; @@ -346,43 +368,32 @@ protected LockedDocumentMap beginProtected(final DBBroker broker, final Map parameters) throws XPathException, IOException, PermissionDeniedException { - final XQuery xquery = broker.getBrokerPool().getXQueryService(); - final XQueryPool pool = broker.getBrokerPool().getXQueryPool(); - @Nullable CompiledXQuery compiled = null; - @Nullable XQueryContext context = null; - - try { - compiled = pool.borrowCompiledXQuery(broker, source); - - if (compiled == null) { - context = new XQueryContext(broker.getBrokerPool()); - } else { - context = compiled.getContext(); - context.prepareForReuse(); - } + private static XQueryUtil.CompilationResult compile(final DBBroker broker, final Source source, final Map parameters) throws XPathException, PermissionDeniedException, IOException { + final ConsumerE preCompilationContext = xqueryContext -> { final String base = (String) parameters.get(RpcAPI.BASE_URI); if (base != null) { - context.setBaseURI(new AnyURIValue(base)); + xqueryContext.setBaseURI(new AnyURIValue(base)); } final String moduleLoadPath = (String) parameters.get(RpcAPI.MODULE_LOAD_PATH); if (moduleLoadPath != null) { - context.setModuleLoadPath(moduleLoadPath); + xqueryContext.setModuleLoadPath(moduleLoadPath); } final Map namespaces = (Map) parameters.get(RpcAPI.NAMESPACES); if (namespaces != null && !namespaces.isEmpty()) { - context.declareNamespaces(namespaces); + xqueryContext.declareNamespaces(namespaces); } final Object[] staticDocuments = (Object[]) parameters.get(RpcAPI.STATIC_DOCUMENTS); @@ -393,45 +404,27 @@ private CompiledXQuery compile(final DBBroker broker, final Source source, final XmldbURI next = XmldbURI.xmldbUriFor((String) staticDocuments[i]); d[i] = next; } - context.setStaticallyKnownDocuments(d); + xqueryContext.setStaticallyKnownDocuments(d); } catch (final URISyntaxException e) { throw new XPathException((Expression) null, e); } - } else if (context.isBaseURIDeclared()) { - context.setStaticallyKnownDocuments(new XmldbURI[]{context.getBaseURI().toXmldbURI()}); - } - - if (compiled == null) { - compiled = xquery.compile(context, source); - } else { - compiled.getContext().updateContext(context); - context.getWatchDog().reset(); + } else if (xqueryContext.isBaseURIDeclared()) { + xqueryContext.setStaticallyKnownDocuments(new XmldbURI[]{ xqueryContext.getBaseURI().toXmldbURI()} ); } + }; - return compiled; - - } catch (final XPathException | IOException | PermissionDeniedException e) { - if (context != null) { - context.runCleanupTasks(); - } - if (compiled != null) { - pool.returnCompiledXQuery(source, compiled); - } - throw e; - } + return XQueryUtil.compile(broker, source, false, true, preCompilationContext); } @Override public String printDiagnostics(final String query, final Map parameters) throws EXistException, PermissionDeniedException { final Source source = new StringSource(query); return withDb((broker, transaction) -> { - try { - return this.compileQuery(broker, transaction, source, parameters).apply(compiledQuery -> { - try (final StringBuilderWriter writer = new StringBuilderWriter()) { - compiledQuery.dump(writer); - return writer.toString(); - } - }); + try (final XQueryUtil.CompilationResult compilationResult = compile(broker, source, parameters)) { + try (final StringBuilderWriter writer = new StringBuilderWriter()) { + compilationResult.dump(writer); + return writer.toString(); + } } catch (final XPathException e) { throw new EXistException(e); } @@ -484,14 +477,11 @@ public int executeQuery(final byte[] xpath, final String encoding, final Map parameters) throws EXistException, PermissionDeniedException { return withDb((broker, transaction) -> { final Source source = new StringSource(xpath); - final long startTime = System.currentTimeMillis(); try { - final QueryResult result = this.compileQuery(broker, transaction, source, parameters).apply(compiledQuery -> doQuery(broker, compiledQuery, null, parameters)); - if (result.hasErrors()) { - throw new EXistException(result.getException()); - } - result.queryTime = System.currentTimeMillis() - startTime; - return factory.resultSets.add(result); + // NOTE(AR) do not call CachedQueryResult.close() here as it will be cached in queryResultToTypedRpcResponse + final XQueryUtil.CompilationResult compiled = compile(broker, source, parameters); + final CachedQueryResult result = doQuery(broker, compiled, null, parameters); + return factory.cachedResultSets.add(result); } catch (final XPathException e) { throw new EXistException(e); } @@ -808,7 +798,7 @@ public Map getDocumentData(final String docName, final Map MAX_DOWNLOAD_CHUNK_SIZE) { offset = firstChunk.length; - final int handle = factory.resultSets.add(new CachedContentFile(tempFile, filePool::returnObject)); + final int handle = factory.cachedResultSets.add(new CachedContentFile(tempFile, filePool::returnObject)); result.put("handle", Integer.toString(handle)); result.put("supports-long-offset", Boolean.TRUE); } else { @@ -840,10 +830,10 @@ private byte[] getChunk(final ContentFile file, final int offset) throws IOExcep @Override public Map getNextChunk(final String handle, final int offset) - throws EXistException, PermissionDeniedException { + throws EXistException { try { final int resultId = Integer.parseInt(handle); - final CachedContentFile sr = factory.resultSets.getCachedContentFile(resultId); + final CachedContentFile sr = factory.cachedResultSets.getCachedContentFile(resultId); if (sr == null) { throw new EXistException("Invalid handle specified"); @@ -853,7 +843,7 @@ public Map getNextChunk(final String handle, final int offset) final ContentFile tempFile = sr.getResult(); if (offset <= 0 || offset > tempFile.size()) { - factory.resultSets.remove(resultId); + factory.cachedResultSets.remove(resultId); throw new EXistException("No more data available"); } final byte[] chunk = getChunk(tempFile, offset); @@ -863,9 +853,10 @@ public Map getNextChunk(final String handle, final int offset) result.put("data", chunk); result.put("handle", handle); if (nextChunk > Integer.MAX_VALUE || nextChunk >= tempFile.size()) { - factory.resultSets.remove(resultId); + factory.cachedResultSets.remove(resultId); result.put("offset", 0); } else { + sr.touch(); result.put("offset", nextChunk); } return result; @@ -876,10 +867,10 @@ public Map getNextChunk(final String handle, final int offset) @Override public Map getNextExtendedChunk(final String handle, final String offset) - throws EXistException, PermissionDeniedException { + throws EXistException { try { final int resultId = Integer.parseInt(handle); - final CachedContentFile sr = factory.resultSets.getCachedContentFile(resultId); + final CachedContentFile sr = factory.cachedResultSets.getCachedContentFile(resultId); if (sr == null) { throw new EXistException("Invalid handle specified"); @@ -890,7 +881,7 @@ public Map getNextExtendedChunk(final String handle, final Strin final long longOffset = Long.parseLong(offset); if (longOffset < 0 || longOffset > tempFile.size()) { - factory.resultSets.remove(resultId); + factory.cachedResultSets.remove(resultId); throw new EXistException("No more data available"); } final byte[] chunk = getChunk(tempFile, (int)longOffset); @@ -900,9 +891,10 @@ public Map getNextExtendedChunk(final String handle, final Strin result.put("data", chunk); result.put("handle", handle); if (nextChunk >= tempFile.size()) { - factory.resultSets.remove(resultId); + factory.cachedResultSets.remove(resultId); result.put("offset", Long.toString(0)); } else { + sr.touch(); result.put("offset", Long.toString(nextChunk)); } return result; @@ -1133,15 +1125,16 @@ private String createResourceId(final XmldbURI collUri) throws EXistException, P @Override public int getHits(final int resultId) throws EXistException { - final QueryResult qr = factory.resultSets.getResult(resultId); + final CachedQueryResult qr = factory.cachedResultSets.getResult(resultId); if (qr == null) { throw new EXistException("result set unknown or timed out"); } qr.touch(); - if (qr.result == null) { + @Nullable final Sequence result = qr.getResult(); + if (result == null) { return 0; } - return qr.result.getItemCount(); + return result.getItemCount(); } @Override @@ -1558,7 +1551,7 @@ private boolean parseLocal(final String localFile, final XmldbURI docUri, final SupplierE sourceSupplier; try { final int handle = Integer.parseInt(localFile); - final SerializedResult sr = factory.resultSets.getSerializedResult(handle); + final SerializedResult sr = factory.cachedResultSets.getSerializedResult(handle); if (sr == null) { // NOTE: early release of Collection lock inline with Asymmetrical Locking scheme collection.close(); @@ -1569,7 +1562,7 @@ private boolean parseLocal(final String localFile, final XmldbURI docUri, final sourceSupplier = () -> { final FileInputSource source = new FileInputSource(sr.result); sr.result = null; // de-reference the temp file in the SerializeResult, so it is not re-claimed before we need it - factory.resultSets.remove(handle); + factory.cachedResultSets.remove(handle); return source; }; } catch (final NumberFormatException nfe) { @@ -1655,7 +1648,7 @@ public String upload(final byte[] chunk, final int length, @Nullable String file // create temporary file tempFile = temporaryFileManager().getTemporaryFile(); - final int handle = factory.resultSets.add(new SerializedResult(tempFile)); + final int handle = factory.cachedResultSets.add(new SerializedResult(tempFile)); fileName = Integer.toString(handle); } else { if(LOG.isDebugEnabled()) { @@ -1667,7 +1660,7 @@ public String upload(final byte[] chunk, final int length, @Nullable String file try { final int handle = Integer.parseInt(fileName); - final SerializedResult sr = factory.resultSets.getSerializedResult(handle); + final SerializedResult sr = factory.cachedResultSets.getSerializedResult(handle); if (sr == null) { throw new EXistException("Invalid handle specified"); } @@ -1753,8 +1746,8 @@ public Map compile(final String query, final Map return withDb((broker, transaction) -> { final Map ret = new HashMap<>(); - try { - compileQuery(broker, transaction, source, parameters).apply(compiledQuery -> null); + try (final XQueryUtil.CompilationResult compiled = compile(broker, source, parameters)) { + // no-op as we are just testing query compilation, and reporting errors if it fails } catch (final XPathException e) { setErrorInformation(ret, e); } @@ -1793,29 +1786,21 @@ private static Map qnameToMap(final Map map, fin return map; } - public String query(final String xpath, final int howmany, final int start, - final Map parameters) throws EXistException, PermissionDeniedException { - + public String query(final String xpath, final int howmany, final int start, final Map parameters) throws EXistException, PermissionDeniedException { final Source source = new StringSource(xpath); - return withDb((broker, transaction) -> { - final long startTime = System.currentTimeMillis(); - try { - final QueryResult qr = this.compileQuery(broker, transaction, source, parameters).apply(compiled -> doQuery(broker, compiled, null, parameters)); - if (qr == null) { + try (final XQueryUtil.CompilationResult compiled = compile(broker, source, parameters); + final CachedQueryResult result = doQuery(broker, compiled, null, parameters)) { + + if (result.getResult() == null) { return "\n" + EXIST_RESULT_XMLNS_EXIST + Namespaces.EXIST_NS + "\" " + "hitCount=\"0\"/>"; } - try { - if (qr.hasErrors()) { - throw qr.getException(); - } - return printAll(broker, qr.result, howmany, start, parameters, (System.currentTimeMillis() - startTime)); - } finally { - qr.close(); - } + + return printAll(broker, result.getResult(), howmany, start, parameters, result.totalTime()); + } catch (final XPathException e) { throw new EXistException(e); } @@ -1884,30 +1869,34 @@ private Map queryP(final String xpath, final XmldbURI docUri, nodes = null; } + final XQueryUtil.CompilationResult compiled; try { - final Map rpcResponse = this.>compileQuery(broker, transaction, source, parameters) - .apply(compiledQuery -> queryResultToRpcResponse(startTime, doQuery(broker, compiledQuery, nodes, parameters), sortBy)); - return rpcResponse; + compiled = compile(broker, source, parameters); } catch (final XPathException e) { throw new EXistException(e); } + + try { + // NOTE(AR) do not call CachedQueryResult.close() here as it will be cached in queryResultToRpcResponse + final CachedQueryResult result = doQuery(broker, compiled, nodes, parameters); + return queryResultToRpcResponse(result, sortBy); + } catch (final XPathException e) { + final Map ret = new HashMap<>(); + setErrorInformation(ret, e); + return ret; + } }); } - private Map queryResultToRpcResponse(final long startTime, final QueryResult queryResult, final Optional sortBy) throws XPathException { + private Map queryResultToRpcResponse(final CachedQueryResult queryResult, final Optional sortBy) throws XPathException { final Map ret = new HashMap<>(); - if (queryResult == null) { - return ret; - } - if (queryResult.hasErrors()) { - // return an error description - final XPathException e = queryResult.getException(); - setErrorInformation(ret, e); + @Nullable Sequence resultSeq = queryResult.getResult(); + + if (resultSeq == null) { return ret; } - Sequence resultSeq = queryResult.result; if (LOG.isDebugEnabled()) { LOG.debug("found {}", resultSeq.getItemCount()); } @@ -1919,40 +1908,34 @@ private Map queryResultToRpcResponse(final long startTime, final } final List result = new ArrayList<>(); - if (resultSeq != null) { - final SequenceIterator i = resultSeq.iterate(); - if (i != null) { - while (i.hasNext()) { - final Item next = i.nextItem(); - if (Type.subTypeOf(next.getType(), Type.NODE)) { - final List entry = new ArrayList<>(); - if (((NodeValue) next).getImplementationType() == NodeValue.PERSISTENT_NODE) { - final NodeProxy p = (NodeProxy) next; - entry.add(p.getOwnerDocument().getURI().toString()); - entry.add(p.getNodeId().toString()); - } else { - entry.add("temp_xquery/" + next.hashCode()); - entry.add(String.valueOf(((NodeImpl) next).getNodeNumber())); - } - result.add(entry); - - } else if (Type.subTypeOf(next.getType(), Type.MAP) || Type.subTypeOf(next.getType(), Type.ARRAY)) { - result.add(next.toString()); - + final SequenceIterator i = resultSeq.iterate(); + if (i != null) { + while (i.hasNext()) { + final Item next = i.nextItem(); + if (Type.subTypeOf(next.getType(), Type.NODE)) { + final List entry = new ArrayList<>(); + if (((NodeValue) next).getImplementationType() == NodeValue.PERSISTENT_NODE) { + final NodeProxy p = (NodeProxy) next; + entry.add(p.getOwnerDocument().getURI().toString()); + entry.add(p.getNodeId().toString()); } else { - result.add(next.getStringValue()); + entry.add("temp_xquery/" + next.hashCode()); + entry.add(String.valueOf(((NodeImpl) next).getNodeNumber())); } + result.add(entry); + + } else if (Type.subTypeOf(next.getType(), Type.MAP) || Type.subTypeOf(next.getType(), Type.ARRAY)) { + result.add(next.toString()); + + } else { + result.add(next.getStringValue()); } - } else if (LOG.isDebugEnabled()) { - LOG.debug("sequence iterator is null. Should not"); } } else if (LOG.isDebugEnabled()) { - LOG.debug("result sequence is null. Skipping it..."); + LOG.debug("sequence iterator is null. Should not"); } - queryResult.result = resultSeq; - queryResult.queryTime = (System.currentTimeMillis() - startTime); - final int id = factory.resultSets.add(queryResult); + final int id = factory.cachedResultSets.add(queryResult); ret.put("id", id); ret.put("hash", queryResult.hashCode()); ret.put("results", result); @@ -2001,13 +1984,22 @@ private Map queryPT(final String xquery, final XmldbURI docUri, nodes = null; } + final XQueryUtil.CompilationResult compiled; try { - final Map rpcResponse = this.>compileQuery(broker, transaction, source, parameters) - .apply(compiledQuery -> queryResultToTypedRpcResponse(startTime, getXdmSerializationOptions(compiledQuery.getContext()), doQuery(broker, compiledQuery, nodes, parameters), sortBy)); - return rpcResponse; + compiled = compile(broker, source, parameters); } catch (final XPathException e) { throw new EXistException(e); } + + try { + // NOTE(AR) do not call CachedQueryResult.close() here as it will be cached in queryResultToTypedRpcResponse + final CachedQueryResult result = doQuery(broker, compiled, nodes, parameters, true); + return queryResultToTypedRpcResponse(result, sortBy); + } catch (final XPathException e) { + final Map ret = new HashMap<>(); + setErrorInformation(ret, e); + return ret; + } }); } @@ -2017,20 +2009,15 @@ private Properties getXdmSerializationOptions(final XQueryContext context) throw return properties; } - private Map queryResultToTypedRpcResponse(final long startTime, final Properties xdmSerializationOptions, final QueryResult queryResult, final Optional sortBy) throws XPathException { + private Map queryResultToTypedRpcResponse(final CachedQueryResult queryResult, final Optional sortBy) throws XPathException { final Map ret = new HashMap<>(); - if (queryResult == null) { - return ret; - } - if (queryResult.hasErrors()) { - // return an error description - final XPathException e = queryResult.getException(); - setErrorInformation(ret, e); + @Nullable Sequence resultSeq = queryResult.getResult(); + + if (resultSeq == null) { return ret; } - Sequence resultSeq = queryResult.result; if (LOG.isDebugEnabled()) { LOG.debug("found {}", resultSeq.getItemCount()); } @@ -2042,39 +2029,33 @@ private Map queryResultToTypedRpcResponse(final long startTime, } final List> result = new ArrayList<>(); - if (resultSeq != null) { - final SequenceIterator i = resultSeq.iterate(); - if (i != null) { - while (i.hasNext()) { - final Item next = i.nextItem(); - final Map entry; - if (Type.subTypeOf(next.getType(), Type.NODE)) { - entry = nodeMap(xdmSerializationOptions, next); - } else { - entry = atomicMap(next); - } + final SequenceIterator i = resultSeq.iterate(); + if (i != null) { + while (i.hasNext()) { + final Item next = i.nextItem(); + final Map entry; + if (Type.subTypeOf(next.getType(), Type.NODE)) { + entry = nodeMap(queryResult.getSerialization(), next); + } else { + entry = atomicMap(next); + } - if(entry != null) { - result.add(entry); - } + if(entry != null) { + result.add(entry); } - } else if (LOG.isDebugEnabled()) { - LOG.debug("sequence iterator is null. Should not"); } } else if (LOG.isDebugEnabled()) { - LOG.debug("result sequence is null. Skipping it..."); + LOG.debug("sequence iterator is null. Should not"); } - queryResult.result = resultSeq; - queryResult.queryTime = (System.currentTimeMillis() - startTime); - final int id = factory.resultSets.add(queryResult); + final int id = factory.cachedResultSets.add(queryResult); ret.put("id", id); ret.put("hash", queryResult.hashCode()); ret.put("results", result); return ret; } - private @Nullable Map nodeMap(final Properties xdmSerializationOptions, final Item item) { + private @Nullable Map nodeMap(@Nullable final Properties xdmSerializationOptions, final Item item) { final Map result; if (item instanceof NodeValue && @@ -2127,63 +2108,81 @@ private Map atomicMap(final Item item) throws XPathException { @Deprecated @Override public Map execute(final String pathToQuery, final Map parameters) throws EXistException, PermissionDeniedException { - final long startTime = System.currentTimeMillis(); - final Optional sortBy = Optional.ofNullable(parameters.get(RpcAPI.SORT_EXPR)).map(Object::toString); - return this.>readDocument(XmldbURI.createInternal(pathToQuery)).apply((document, broker, transaction) -> { - final BinaryDocument xquery = (BinaryDocument) document; - if (xquery.getResourceType() != DocumentImpl.BINARY_FILE) { - throw new EXistException("Document " + pathToQuery + " is not a binary resource"); - } + return withDb((broker, transaction) -> { - if (!xquery.getPermissions().validate(user, Permission.READ | Permission.EXECUTE)) { - throw new PermissionDeniedException("Insufficient privileges to access resource"); - } + final XQueryUtil.CompilationResult compiledXquery = this.readDocument(broker, transaction, XmldbURI.createInternal(pathToQuery)).apply((document, broker1, transaction1) -> { + if (document.getResourceType() != DocumentImpl.BINARY_FILE) { + throw new EXistException("Document " + pathToQuery + " is not a binary resource"); + } + + if (!document.getPermissions().validate(user, Permission.READ | Permission.EXECUTE)) { + throw new PermissionDeniedException("Insufficient privileges to access resource"); + } + + final Source source = new DBSource(broker1.getBrokerPool(), (BinaryDocument) document, true); + try { + return compile(broker1, source, parameters); + } catch (final XPathException e) { + throw new EXistException(e); + } + }); - final Source source = new DBSource(broker, xquery, true); + // NOTE(AR) query is now compiled so we can release the readDocument lock eagerly above try { - final Map rpcResponse = this.>compileQuery(broker, transaction, source, parameters) - .apply(compiledQuery -> queryResultToRpcResponse(startTime, doQuery(broker, compiledQuery, null, parameters), sortBy)); - return rpcResponse; + // NOTE(AR) do not call CachedQueryResult.close() here as it will be cached in queryResultToRpcResponse + final CachedQueryResult queryResult = doQuery(broker, compiledXquery, null, parameters); + return queryResultToRpcResponse(queryResult, sortBy); } catch (final XPathException e) { - throw new EXistException(e); + final Map ret = new HashMap<>(); + setErrorInformation(ret, e); + return ret; } }); } @Override public Map executeT(final String pathToQuery, final Map parameters) throws EXistException, PermissionDeniedException { - final long startTime = System.currentTimeMillis(); - final Optional sortBy = Optional.ofNullable(parameters.get(RpcAPI.SORT_EXPR)).map(Object::toString); - return this.>readDocument(XmldbURI.createInternal(pathToQuery)).apply((document, broker, transaction) -> { - final BinaryDocument xquery = (BinaryDocument) document; - if (xquery.getResourceType() != DocumentImpl.BINARY_FILE) { - throw new EXistException("Document " + pathToQuery + " is not a binary resource"); - } + return withDb((broker, transaction) -> { - if (!xquery.getPermissions().validate(user, Permission.READ | Permission.EXECUTE)) { - throw new PermissionDeniedException("Insufficient privileges to access resource"); - } + final XQueryUtil.CompilationResult compiledXquery = this.readDocument(broker, transaction, XmldbURI.createInternal(pathToQuery)).apply((document, broker1, transaction1) -> { + if (document.getResourceType() != DocumentImpl.BINARY_FILE) { + throw new EXistException("Document " + pathToQuery + " is not a binary resource"); + } + + if (!document.getPermissions().validate(user, Permission.READ | Permission.EXECUTE)) { + throw new PermissionDeniedException("Insufficient privileges to access resource"); + } + + final Source source = new DBSource(broker1.getBrokerPool(), (BinaryDocument) document, true); + try { + return compile(broker1, source, parameters); + } catch (final XPathException e) { + throw new EXistException(e); + } + }); - final Source source = new DBSource(broker, xquery, true); + // NOTE(AR) query is now compiled so we can release the readDocument lock eagerly above try { - final Map rpcResponse = this.>compileQuery(broker, transaction, source, parameters) - .apply(compiledQuery -> queryResultToTypedRpcResponse(startTime, getXdmSerializationOptions(compiledQuery.getContext()), doQuery(broker, compiledQuery, null, parameters), sortBy)); - return rpcResponse; + // NOTE(AR) do not call CachedQueryResult.close() here as it will be cached in queryResultToTypedRpcResponse + final CachedQueryResult queryResult = doQuery(broker, compiledXquery, null, parameters, true); + return queryResultToTypedRpcResponse(queryResult, sortBy); } catch (final XPathException e) { - throw new EXistException(e); + final Map ret = new HashMap<>(); + setErrorInformation(ret, e); + return ret; } }); } @Override public boolean releaseQueryResult(final int handle) { - factory.resultSets.remove(handle); + factory.cachedResultSets.remove(handle); if (LOG.isDebugEnabled()) { LOG.debug("removed query result with handle {}", handle); } @@ -2192,7 +2191,7 @@ public boolean releaseQueryResult(final int handle) { @Override public boolean releaseQueryResult(final int handle, final int hash) { - factory.resultSets.remove(handle, hash); + factory.cachedResultSets.remove(handle, hash); if (LOG.isDebugEnabled()) { LOG.debug("removed query result with handle {}", handle); } @@ -2341,7 +2340,7 @@ public Map retrieveFirstChunk(final String docName, final String if (tempFile.size() > MAX_DOWNLOAD_CHUNK_SIZE) { offset = firstChunk.length; - final int handle = factory.resultSets.add(new CachedContentFile(tempFile, filePool::returnObject)); + final int handle = factory.cachedResultSets.add(new CachedContentFile(tempFile, filePool::returnObject)); result.put("handle", Integer.toString(handle)); result.put("supports-long-offset", Boolean.TRUE); } else { @@ -2380,20 +2379,22 @@ public byte[] retrieve(final int resultId, final int num, final Map parameters) throws EXistException, PermissionDeniedException { return withDb((broker, transaction) -> { - final QueryResult qr = factory.resultSets.getResult(resultId); + final CachedQueryResult qr = factory.cachedResultSets.getResult(resultId); if (qr == null) { throw new EXistException("result set unknown or timed out"); } qr.touch(); - final Item item = qr.result.itemAt(num); + final Item item = qr.getResult().itemAt(num); if (item == null) { throw new EXistException("index out of range"); } if (Type.subTypeOf(item.getType(), Type.NODE)) { final NodeValue nodeValue = (NodeValue) item; - for (final Map.Entry entry : qr.serialization.entrySet()) { - parameters.put(entry.getKey().toString(), entry.getValue().toString()); + if (qr.getSerialization() != null) { + for (final Map.Entry entry : qr.getSerialization().entrySet()) { + parameters.put(entry.getKey().toString(), entry.getValue().toString()); + } } try (final StringBuilderWriter writer = new StringBuilderWriter()) { final Properties properties = toProperties(parameters); @@ -2417,19 +2418,21 @@ public Map retrieveFirstChunk(final int resultId, final int num, return withDb((broker, transaction) -> { - final QueryResult qr = factory.resultSets.getResult(resultId); + final CachedQueryResult qr = factory.cachedResultSets.getResult(resultId); if (qr == null) { throw new EXistException("result set unknown or timed out: " + resultId); } qr.touch(); - final Item item = qr.result.itemAt(num); + final Item item = qr.getResult().itemAt(num); if (item == null) { throw new EXistException("index out of range"); } final Properties serializationProperties = toProperties(parameters); - for (final Map.Entry entry : qr.serialization.entrySet()) { - serializationProperties.put(entry.getKey().toString(), entry.getValue().toString()); + if (qr.getSerialization() != null) { + for (final Map.Entry entry : qr.getSerialization().entrySet()) { + serializationProperties.put(entry.getKey().toString(), entry.getValue().toString()); + } } final Map result = new HashMap<>(); @@ -2466,7 +2469,7 @@ public Map retrieveFirstChunk(final int resultId, final int num, if (tempFile.size() > MAX_DOWNLOAD_CHUNK_SIZE) { offset = firstChunk.length; - final int handle = factory.resultSets.add(new CachedContentFile(tempFile, filePool::returnObject)); + final int handle = factory.cachedResultSets.add(new CachedContentFile(tempFile, filePool::returnObject)); result.put("handle", Integer.toString(handle)); result.put("supports-long-offset", Boolean.TRUE); } else { @@ -2488,7 +2491,7 @@ public byte[] retrieveAll(final int resultId, final Map paramete private String retrieveAllAsString(final int resultId, final Map parameters) throws EXistException, PermissionDeniedException { return withDb((broker, transaction) -> { - final QueryResult qr = factory.resultSets.getResult(resultId); + final CachedQueryResult qr = factory.cachedResultSets.getResult(resultId); if (qr == null) { throw new EXistException("result set unknown or timed out"); } @@ -2503,12 +2506,12 @@ private String retrieveAllAsString(final int resultId, final Map handler.startPrefixMapping("exist", Namespaces.EXIST_NS); handler.startPrefixMapping("xs", Namespaces.SCHEMA_NS); final AttributesImpl attribs = new AttributesImpl(); - attribs.addAttribute("", "hitCount", "hitCount", "CDATA", Integer.toString(qr.result.getItemCount())); + attribs.addAttribute("", "hitCount", "hitCount", "CDATA", Integer.toString(qr.getResult().getItemCount())); handler.startElement(Namespaces.EXIST_NS, "result", "exist:result", attribs); Item current; char[] value; try { - for (final SequenceIterator i = qr.result.iterate(); i.hasNext(); ) { + for (final SequenceIterator i = qr.getResult().iterate(); i.hasNext(); ) { current = i.nextItem(); if (Type.subTypeOf(current.getType(), Type.NODE)) { @@ -2544,13 +2547,15 @@ public Map retrieveAllFirstChunk(final int resultId, final Map { - final QueryResult qr = factory.resultSets.getResult(resultId); + final CachedQueryResult qr = factory.cachedResultSets.getResult(resultId); if (qr == null) { throw new EXistException("result set unknown or timed out"); } qr.touch(); - for (final Map.Entry entry : qr.serialization.entrySet()) { - parameters.put(entry.getKey().toString(), entry.getValue().toString()); + if (qr.getSerialization() != null) { + for (final Map.Entry entry : qr.getSerialization().entrySet()) { + parameters.put(entry.getKey().toString(), entry.getValue().toString()); + } } final SAXSerializer handler = (SAXSerializer) SerializerPool.getInstance().borrowObject(SAXSerializer.class); try { @@ -2577,7 +2582,7 @@ public Map retrieveAllFirstChunk(final int resultId, final Map retrieveAllFirstChunk(final int resultId, final Map retrieveAllFirstChunk(final int resultId, final Map MAX_DOWNLOAD_CHUNK_SIZE) { offset = firstChunk.length; - final int handle = factory.resultSets.add(new CachedContentFile(tempFile, filePool::returnObject)); + final int handle = factory.cachedResultSets.add(new CachedContentFile(tempFile, filePool::returnObject)); result.put("handle", Integer.toString(handle)); result.put("supports-long-offset", Boolean.TRUE); } else { @@ -3085,23 +3090,16 @@ public Map summary(final String xpath) throws EXistException, Pe final long startTime = System.currentTimeMillis(); final Map parameters = new HashMap<>(); - try { - final QueryResult qr = this.compileQuery(broker, transaction, source, parameters).apply(compiledQuery -> doQuery(broker, compiledQuery, null, parameters)); - if (qr == null) { - return new HashMap<>(); - } - try { - if (qr.hasErrors()) { - throw qr.getException(); - } - if (qr.result == null) { - return summaryToMap(qr.queryTime, null, null, null); + try (final XQueryUtil.CompilationResult compiled = compile(broker, source, parameters); + final CachedQueryResult result = doQuery(broker, compiled, null, parameters)) { + + if (result.getResult() == null) { + return summaryToMap(result, null, null); } - final Tuple2, java.util.Collection> summary = summarise(qr.result); - return summaryToMap(System.currentTimeMillis() - startTime, qr.result, summary._1, summary._2); - } finally { - qr.close(); - } + + final Tuple2, java.util.Collection> summary = summarise(result.getResult()); + return summaryToMap(result, summary._1, summary._2); + } catch (final XPathException e) { throw new EXistException(e); } @@ -3109,17 +3107,19 @@ public Map summary(final String xpath) throws EXistException, Pe } public Map summary(final int resultId) throws EXistException, XPathException { - final QueryResult qr = factory.resultSets.getResult(resultId); + final CachedQueryResult qr = factory.cachedResultSets.getResult(resultId); if (qr == null) { throw new EXistException("result set unknown or timed out"); } + qr.touch(); - if (qr.result == null) { - return summaryToMap(qr.queryTime, null, null, null); + + if (qr.getResult() == null) { + return summaryToMap(qr, null, null); } - final Tuple2, java.util.Collection> summary = summarise(qr.result); - return summaryToMap(qr.queryTime, qr.result, summary._1, summary._2); + final Tuple2, java.util.Collection> summary = summarise(qr.getResult()); + return summaryToMap(qr, summary._1, summary._2); } private Tuple2, java.util.Collection> summarise(final Sequence results) throws XPathException { @@ -3159,35 +3159,39 @@ private Tuple2, java.util.Collection(nodeCounts.values(), doctypeCounts.values()); } - private Map summaryToMap(final long queryTime, @Nullable final Sequence results, + private Map summaryToMap(final CachedQueryResult results, @Nullable final java.util.Collection nodeCounts, @Nullable final java.util.Collection doctypeCounts) { final Map result = new HashMap<>(); - result.put("queryTime", queryTime); + result.put("queryTime", results.totalTime()); - if (results == null) { + if (results.getResult() == null) { result.put("hits", 0); return result; } - result.put("hits", results.getItemCount()); + result.put("hits", results.getResult().getItemCount()); - final List documents = new ArrayList<>(); - for (final NodeCount nodeCount : nodeCounts) { - final List hitsByDoc = new ArrayList<>(); - hitsByDoc.add(nodeCount.doc.getFileURI().toString()); - hitsByDoc.add(nodeCount.doc.getDocId()); - hitsByDoc.add(nodeCount.count); - documents.add(hitsByDoc); + final List> documents = new ArrayList<>(); + if (nodeCounts != null) { + for (final NodeCount nodeCount : nodeCounts) { + final List hitsByDoc = new ArrayList<>(); + hitsByDoc.add(nodeCount.doc.getFileURI().toString()); + hitsByDoc.add(nodeCount.doc.getDocId()); + hitsByDoc.add(nodeCount.count); + documents.add(hitsByDoc); + } } result.put("documents", documents); - final List dtypes = new ArrayList<>(); - for (final DoctypeCount docTemp : doctypeCounts) { - final List hitsByType = new ArrayList<>(); - hitsByType.add(docTemp.doctype.getName()); - hitsByType.add(docTemp.count); - dtypes.add(hitsByType); + final List> dtypes = new ArrayList<>(); + if(doctypeCounts != null) { + for (final DoctypeCount docTemp : doctypeCounts) { + final List hitsByType = new ArrayList<>(); + hitsByType.add(docTemp.doctype.getName()); + hitsByType.add(docTemp.count); + dtypes.add(hitsByType); + } } result.put("doctypes", dtypes); @@ -3779,7 +3783,7 @@ public void runCommand(final XmldbURI collectionURI, final List params) @Override public String restore(final String newAdminPassword, final String localFile, final boolean overwriteApps) throws EXistException { final int handle = Integer.parseInt(localFile); - final SerializedResult sr = factory.resultSets.getSerializedResult(handle); + final SerializedResult sr = factory.cachedResultSets.getSerializedResult(handle); if (sr == null) { throw new EXistException("Invalid handle specified"); } @@ -3790,7 +3794,7 @@ public String restore(final String newAdminPassword, final String localFile, fin final Path backupFile = sr.result; try { sr.result = null; // de-reference the temp file in the SerializeResult, so it is not re-claimed before we need it - factory.resultSets.remove(handle); + factory.cachedResultSets.remove(handle); withDb((broker, transaction) -> { final Restore restore = new Restore(); @@ -3934,29 +3938,6 @@ private boolean useCompression(final Map parameters) { return Optional.ofNullable(parameters.get(EXistOutputKeys.COMPRESS_OUTPUT)).map(c -> c.toString().equalsIgnoreCase("yes")).orElse(false); } - /** - * Takes a query from the pool or compiles a new one - */ - private Function3E, R, EXistException, PermissionDeniedException, XPathException> compileQuery(final DBBroker broker, final Txn transaction, final Source source, final Map parameters) throws EXistException, PermissionDeniedException { - return compiledOp -> { - final XQueryPool pool = broker.getBrokerPool().getXQueryPool(); - CompiledXQuery compiled = null; - try { - compiled = compile(broker, source, parameters); - return compiledOp.apply(compiled); - } catch (final IOException e) { - throw new EXistException(e); - } finally { - if (compiled != null) { - if (compiled.getContext() != null) { - compiled.getContext().runCleanupTasks(); - } - pool.returnCompiledXQuery(source, compiled); - } - } - }; - } - /** * Higher-order function for performing read locked operations on a collection * diff --git a/exist-core/src/main/java/org/exist/xmlrpc/XmldbRequestProcessorFactory.java b/exist-core/src/main/java/org/exist/xmlrpc/XmldbRequestProcessorFactory.java index 31058d0caf..843e16ea4c 100644 --- a/exist-core/src/main/java/org/exist/xmlrpc/XmldbRequestProcessorFactory.java +++ b/exist-core/src/main/java/org/exist/xmlrpc/XmldbRequestProcessorFactory.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -59,7 +83,7 @@ public class XmldbRequestProcessorFactory implements RequestProcessorFactoryFact private final boolean useDefaultUser; private final BrokerPool brokerPool; private final ContentFilePool contentFilePool; - protected final QueryResultCache resultSets = new QueryResultCache(); + protected final QueryResultCache cachedResultSets = new QueryResultCache(); protected final AtomicLazyVal restoreExecutorService; protected final Map>> restoreTasks = new ConcurrentHashMap<>(); diff --git a/exist-core/src/main/java/org/exist/xmlrpc/function/XmlRpcCompiledXQueryFunction.java b/exist-core/src/main/java/org/exist/xmlrpc/function/XmlRpcCompiledXQueryFunction.java deleted file mode 100644 index 5562717b00..0000000000 --- a/exist-core/src/main/java/org/exist/xmlrpc/function/XmlRpcCompiledXQueryFunction.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xmlrpc.function; - -import org.exist.EXistException; -import org.exist.security.PermissionDeniedException; -import com.evolvedbinary.j8fu.function.Function2E; -import org.exist.xquery.CompiledXQuery; -import org.exist.xquery.XPathException; - -/** - * Specialisation of FunctionE which deals with - * XML-RPC server operations; Predominantly converts exceptions - * from the database into EXistException types - * - * @author Adam Retter - */ -@FunctionalInterface -public interface XmlRpcCompiledXQueryFunction extends Function2E { - - @Override - default R apply(final CompiledXQuery compiledQuery) throws EXistException, PermissionDeniedException { - try { - return applyXmlRpc(compiledQuery); - } catch(final XPathException e) { - throw new EXistException(e); - } - } - - /** - * Signature for lambda function which takes a compiled XQuery - * - * @param compiledQuery The compiled XQuery - * - * @return the result of the function - * - * @throws EXistException if an error occurs with the database - * @throws PermissionDeniedException if the caller has insufficient priviledges - * @throws XPathException if executing the XQuery raises an error - */ - R applyXmlRpc(final CompiledXQuery compiledQuery) throws EXistException, PermissionDeniedException, XPathException; -} diff --git a/exist-core/src/main/java/org/exist/xquery/CompiledXQuery.java b/exist-core/src/main/java/org/exist/xquery/CompiledXQuery.java index 8d95d66675..5f60d15c2f 100644 --- a/exist-core/src/main/java/org/exist/xquery/CompiledXQuery.java +++ b/exist-core/src/main/java/org/exist/xquery/CompiledXQuery.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -26,13 +50,11 @@ import org.exist.source.Source; import org.exist.xquery.value.Item; import org.exist.xquery.value.Sequence; -import org.xmldb.api.base.CompiledExpression; - /** * @author wolf */ -public interface CompiledXQuery extends CompiledExpression { +public interface CompiledXQuery { /** * Reset the compiled expression tree. Discard all diff --git a/exist-core/src/main/java/org/exist/xquery/Context.java b/exist-core/src/main/java/org/exist/xquery/Context.java index d6277cbbcb..14cee7638f 100644 --- a/exist-core/src/main/java/org/exist/xquery/Context.java +++ b/exist-core/src/main/java/org/exist/xquery/Context.java @@ -654,14 +654,14 @@ public interface Context { * * @return DBBroker instance */ - DBBroker getBroker(); + @Nullable DBBroker getBroker(); /** * Get the subject which executes the current query. * * @return subject */ - Subject getSubject(); + @Nullable Subject getSubject(); /** * Get the document builder currently used for creating temporary document fragments. diff --git a/exist-core/src/main/java/org/exist/xquery/ExternalModule.java b/exist-core/src/main/java/org/exist/xquery/ExternalModule.java index be69b88470..b19b1577f2 100644 --- a/exist-core/src/main/java/org/exist/xquery/ExternalModule.java +++ b/exist-core/src/main/java/org/exist/xquery/ExternalModule.java @@ -23,7 +23,6 @@ import org.exist.dom.QName; import org.exist.source.Source; -import org.exist.storage.DBBroker; import java.util.Collection; import java.util.Map; @@ -105,10 +104,9 @@ public interface ExternalModule extends Module { /** * Is this module still valid or should it be reloaded from its source? * - * @param broker the broker to use for checking * @return true if module should be reloaded */ - public boolean moduleIsValid(DBBroker broker); + public boolean moduleIsValid(); /** * Returns the root expression associated with this context. diff --git a/exist-core/src/main/java/org/exist/xquery/ExternalModuleImpl.java b/exist-core/src/main/java/org/exist/xquery/ExternalModuleImpl.java index f8f8eef6a6..ba5ae692d0 100644 --- a/exist-core/src/main/java/org/exist/xquery/ExternalModuleImpl.java +++ b/exist-core/src/main/java/org/exist/xquery/ExternalModuleImpl.java @@ -50,7 +50,6 @@ import org.exist.dom.QName; import org.exist.source.Source; import org.exist.xquery.value.Sequence; -import org.exist.storage.DBBroker; import javax.annotation.Nullable; @@ -311,8 +310,8 @@ public XQueryContext getContext() { } @Override - public boolean moduleIsValid(final DBBroker broker) { - return mSource != null && mSource.isValid(broker) == Source.Validity.VALID; + public boolean moduleIsValid() { + return mSource != null && mSource.isValid() == Source.Validity.VALID; } @Override diff --git a/exist-core/src/main/java/org/exist/xquery/JavaBinding.java b/exist-core/src/main/java/org/exist/xquery/JavaBinding.java index 70384cba03..6830df194c 100644 --- a/exist-core/src/main/java/org/exist/xquery/JavaBinding.java +++ b/exist-core/src/main/java/org/exist/xquery/JavaBinding.java @@ -277,7 +277,14 @@ public static List createReflectiveCall(final int lineNumber // instance method final boolean voidReturnThis = javaClassNameAndBindingParams._2.voidReturnsThis && (void.class == javaReturnType || Void.class == javaReturnType); javaReflectiveCallInvoker = javaArgs -> { - final Object javaResult = method.invoke(javaArgs[0], Arrays.copyOfRange(javaArgs, 1, javaArgs.length)); + final Object instance = javaArgs[0]; + final Class instanceClazz = instance.getClass(); + final Class methodDeclaringClazz = method.getDeclaringClass(); + if (!methodDeclaringClazz.isAssignableFrom(instanceClazz)) { + throw new IllegalArgumentException("Expected an instance of type (or sub-type) of: " + methodDeclaringClazz.getName() + ", but received type: " + instanceClazz.getName()); + } + + final Object javaResult = method.invoke(instance, Arrays.copyOfRange(javaArgs, 1, javaArgs.length)); if (voidReturnThis) { // method returns void and `?void=this` is set, so return this return javaArgs[0]; diff --git a/exist-core/src/main/java/org/exist/xquery/PathExpr.java b/exist-core/src/main/java/org/exist/xquery/PathExpr.java index 83bfcf5344..7c6eb5cf29 100644 --- a/exist-core/src/main/java/org/exist/xquery/PathExpr.java +++ b/exist-core/src/main/java/org/exist/xquery/PathExpr.java @@ -51,7 +51,6 @@ import org.exist.dom.persistent.VirtualNodeSet; import org.exist.xquery.util.ExpressionDumper; import org.exist.xquery.value.*; -import org.xmldb.api.base.CompiledExpression; import java.io.Writer; import java.util.ArrayList; @@ -66,8 +65,7 @@ * @author perig * @author ljo */ -public class PathExpr extends AbstractExpression implements CompiledXQuery, - CompiledExpression, RewritableExpression { +public class PathExpr extends AbstractExpression implements CompiledXQuery, RewritableExpression { protected final static Logger LOG = LogManager.getLogger(PathExpr.class); diff --git a/exist-core/src/main/java/org/exist/xquery/UserDefinedFunction.java b/exist-core/src/main/java/org/exist/xquery/UserDefinedFunction.java index 117d0b5961..23f39d49e8 100644 --- a/exist-core/src/main/java/org/exist/xquery/UserDefinedFunction.java +++ b/exist-core/src/main/java/org/exist/xquery/UserDefinedFunction.java @@ -315,7 +315,12 @@ public List getClosureVariables() { return closureVariables; } - protected Sequence[] getCurrentArguments() { + /** + * Get the current arguments to the user defined function. + * + * @return the current arguments. + */ + public Sequence[] getCurrentArguments() { return currentArguments; } } diff --git a/exist-core/src/main/java/org/exist/xquery/XPathUtil.java b/exist-core/src/main/java/org/exist/xquery/XPathUtil.java index 878da724b1..3e5cb739ce 100644 --- a/exist-core/src/main/java/org/exist/xquery/XPathUtil.java +++ b/exist-core/src/main/java/org/exist/xquery/XPathUtil.java @@ -47,7 +47,6 @@ import java.math.BigDecimal; import java.math.BigInteger; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -56,29 +55,18 @@ import io.lacuna.bifurcan.IMap; import org.exist.dom.memtree.AttrImpl; import org.exist.dom.persistent.AVLTreeNodeSet; -import org.exist.dom.persistent.DocumentImpl; import org.exist.dom.persistent.NodeProxy; import org.exist.dom.memtree.DocumentBuilderReceiver; import org.exist.dom.memtree.MemTreeBuilder; import org.exist.dom.memtree.NodeImpl; -import org.exist.numbering.NodeId; -import org.exist.security.PermissionDeniedException; -import org.exist.storage.DBBroker; import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream; import org.exist.util.serializer.DOMStreamer; import org.exist.util.serializer.SerializerPool; -import org.exist.xmldb.LocalXMLResource; -import org.exist.xmldb.RemoteXMLResource; -import org.exist.xmldb.XmldbURI; import org.exist.xquery.functions.array.ArrayType; import org.exist.xquery.functions.map.MapType; import org.exist.xquery.value.*; import org.w3c.dom.*; import org.xml.sax.SAXException; -import org.xmldb.api.base.ResourceIterator; -import org.xmldb.api.base.ResourceSet; -import org.xmldb.api.base.XMLDBException; -import org.xmldb.api.modules.XMLResource; import javax.annotation.Nullable; @@ -680,21 +668,6 @@ public static final Sequence javaObjectToXPath(final Object obj, final XQueryCon } else if (obj instanceof Sequence) { return (Sequence) obj; - } else if (obj instanceof ResourceSet) { - final Sequence seq = new AVLTreeNodeSet(); - try { - final DBBroker broker = context.getBroker(); - for (final ResourceIterator it = ((ResourceSet) obj).getIterator(); it.hasMoreResources();) { - seq.add(getNode(broker, (XMLResource) it.nextResource(), expression)); - } - } catch (final XMLDBException xe) { - throw new XPathException(expression, "Failed to convert ResourceSet to node: " + xe.getMessage()); - } - return seq; - - } else if (obj instanceof XMLResource) { - return getNode(context.getBroker(), (XMLResource) obj, expression); - } else if (obj instanceof Node) { context.pushDocumentContext(); final DOMStreamer streamer = (DOMStreamer) SerializerPool.getInstance().borrowObject(DOMStreamer.class); @@ -907,50 +880,4 @@ private static Sequence javaArrayToXPath(final Iterable objects, final X } return result; } - - /** - * Converts an XMLResource into a NodeProxy. - * - * @param broker The DBBroker to use to access the database - * @param xres The XMLResource to convert - * @return A NodeProxy for accessing the content represented by xres - * @throws XPathException if an XMLDBException is encountered - */ - public static final NodeProxy getNode(DBBroker broker, XMLResource xres) throws XPathException { - return getNode(broker, xres, null); - } - - /** - * Converts an XMLResource into a NodeProxy. - * - * @param broker The DBBroker to use to access the database - * @param xres The XMLResource to convert - * @param expression the expression from which the resource derives - * @return A NodeProxy for accessing the content represented by xres - * @throws XPathException if an XMLDBException is encountered - */ - public static final NodeProxy getNode(final DBBroker broker, final XMLResource xres, final Expression expression) throws XPathException { - if (xres instanceof LocalXMLResource) { - final LocalXMLResource lres = (LocalXMLResource) xres; - try { - return lres.getNode(); - } catch (final XMLDBException xe) { - throw new XPathException(expression, "Failed to convert LocalXMLResource to node: " + xe.getMessage()); - } - } - - DocumentImpl document; - try { - document = broker.getCollection(XmldbURI.xmldbUriFor(xres.getParentCollection().getName())).getDocument(broker, XmldbURI.xmldbUriFor(xres.getDocumentId())); - } catch (final URISyntaxException xe) { - throw new XPathException(expression, xe); - } catch (final XMLDBException xe) { - throw new XPathException(expression, "Failed to get document for RemoteXMLResource: " + xe.getMessage()); - } catch (final PermissionDeniedException pde) { - throw new XPathException(expression, "Failed to get document: " + pde.getMessage()); - } - final NodeId nodeId = broker.getBrokerPool().getNodeFactory().createFromString(((RemoteXMLResource) xres).getNodeId()); - return new NodeProxy(null, document, nodeId); - - } } diff --git a/exist-core/src/main/java/org/exist/xquery/XQuery.java b/exist-core/src/main/java/org/exist/xquery/XQuery.java index 3b10b65bc8..3b3fce3c5d 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQuery.java +++ b/exist-core/src/main/java/org/exist/xquery/XQuery.java @@ -67,7 +67,6 @@ import org.exist.security.Permission; import org.exist.security.PermissionDeniedException; import org.exist.security.Subject; -import org.exist.source.DBSource; import org.exist.source.FileSource; import org.exist.source.Source; import org.exist.source.StringSource; @@ -229,11 +228,17 @@ public CompiledXQuery compile(final XQueryContext context, final Source source, private CompiledXQuery compile(final XQueryContext context, final Reader reader, final boolean xpointer) throws XPathException, PermissionDeniedException { //check read permission - if (context.getSource() instanceof DBSource) { - ((DBSource) context.getSource()).validate(Permission.READ); + @Nullable final Subject currentSubject = context.getSubject(); + if (currentSubject != null) { + try { + if (!context.getSource().getPermissions().validate(currentSubject, Permission.READ)) { + throw new PermissionDeniedException("Subject '" + currentSubject.getName() + "' does not have read access to resource '" + context.getSource().pathOrShortIdentifier() + "'."); + } + } catch (final IOException e) { + throw new PermissionDeniedException("Subject '" + currentSubject.getName() + "' does not have read access to resource '" + context.getSource().pathOrShortIdentifier() + "'.", e); + } } - //TODO: move XQueryContext.getUserFromHttpSession() here, have to check if servlet.jar is in the classpath //before compiling/executing that code though to avoid a dependency on servlet.jar - reflection? - deliriumsky @@ -376,8 +381,12 @@ public Sequence execute(final DBBroker broker, final CompiledXQuery expression, public Sequence execute(final DBBroker broker, final CompiledXQuery expression, @Nullable final Tuple3, Optional> functionCall, @Nullable Sequence contextSequence, final Properties outputProperties, final boolean resetContext) throws XPathException, PermissionDeniedException { //check execute permissions - if (expression.getContext().getSource() instanceof DBSource) { - ((DBSource) expression.getContext().getSource()).validate(Permission.EXECUTE); + try { + if (!expression.getContext().getSource().getPermissions().validate(broker.getCurrentSubject(), Permission.EXECUTE)) { + throw new PermissionDeniedException("Subject '" + broker.getCurrentSubject().getName() + "' does not have execute access to resource '" + expression.getContext().getSource().pathOrShortIdentifier() + "'."); + } + } catch (final IOException e) { + throw new PermissionDeniedException("Subject '" + broker.getCurrentSubject().getName() + "' does not have execute access to resource '" + expression.getContext().getSource().pathOrShortIdentifier() + "'.", e); } final long start = System.currentTimeMillis(); @@ -405,22 +414,23 @@ public Sequence execute(final DBBroker broker, final CompiledXQuery expression, //if setUid or setGid, become Effective User EffectiveSubject effectiveSubject = null; final Source src = expression.getContext().getSource(); - if(src instanceof DBSource) { - final DBSource dbSrc = (DBSource)src; - final Permission perm = dbSrc.getPermissions(); - - if(perm.isSetUid()) { - if(perm.isSetGid()) { - //setUid and SetGid - effectiveSubject = new EffectiveSubject(perm.getOwner(), perm.getGroup()); - } else { - //just setUid - effectiveSubject = new EffectiveSubject(perm.getOwner()); - } - } else if(perm.isSetGid()) { - //just setGid, so we use the current user as the effective user - effectiveSubject = new EffectiveSubject(callingUser, perm.getGroup()); + final Permission perm; + try { + perm = src.getPermissions(); + } catch (final IOException e) { + throw new PermissionDeniedException(e); + } + if (perm.isSetUid()) { + if(perm.isSetGid()) { + //setUid and SetGid + effectiveSubject = new EffectiveSubject(perm.getOwner(), perm.getGroup()); + } else { + //just setUid + effectiveSubject = new EffectiveSubject(perm.getOwner()); } + } else if (perm.isSetGid()) { + //just setGid, so we use the current user as the effective user + effectiveSubject = new EffectiveSubject(callingUser, perm.getGroup()); } try { diff --git a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java index f8bfb87ad8..ad56571b26 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java +++ b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java @@ -101,7 +101,11 @@ import org.exist.security.Permission; import org.exist.security.PermissionDeniedException; import org.exist.security.Subject; -import org.exist.source.*; +import org.exist.source.DbStoreSource; +import org.exist.source.DbUriSource; +import org.exist.source.FileSource; +import org.exist.source.Source; +import org.exist.source.SourceFactory; import org.exist.stax.ExtendedXMLStreamReader; import org.exist.storage.BrokerPool; import org.exist.storage.DBBroker; @@ -620,7 +624,7 @@ public Optional getRepository() { Source src = repo.get().resolveStoredXQueryModuleFromDb(getBroker(), resolved); if (src != null) { // NOTE(AR) set the location of the module to import relative to this module's load path - so that transient imports of the imported module will resolve correctly! - final Path srcCollectionPath = Paths.get(((DBSource)src).getDocumentPath().getCollectionPath()); + final Path srcCollectionPath = Paths.get(((DbStoreSource)src).getDocumentPath().getCollectionPath()); if (srcCollectionPath.isAbsolute()) { location = srcCollectionPath.toString(); } else { @@ -1710,7 +1714,7 @@ public boolean checkModulesValid() { for (final Module[] modules : allModules.values()) { for (final Module module : modules) { if (!module.isInternalModule()) { - if (!((ExternalModule) module).moduleIsValid(getBroker())) { + if (!((ExternalModule) module).moduleIsValid()) { if (LOG.isDebugEnabled()) { LOG.debug("Module with URI {} has changed and needs to be reloaded", module.getNamespaceURI()); } @@ -2208,7 +2212,7 @@ public Database getDatabase() { } @Override - public DBBroker getBroker() { + public @Nullable DBBroker getBroker() { if (db != null) { return db.getActiveBroker(); } @@ -2220,8 +2224,12 @@ public Configuration getConfiguration() { } @Override - public Subject getSubject() { - return getBroker().getCurrentSubject(); + public @Nullable Subject getSubject() { + @Nullable final DBBroker broker = getBroker(); + if (broker == null) { + return null; + } + return broker.getCurrentSubject(); } /** @@ -2693,58 +2701,70 @@ private Module importModuleFromLocation(final String namespaceURI, @Nullable fin return loadBuiltInModule(namespaceURI, location); } - if (location.startsWith(XmldbURI.XMLDB_URI_PREFIX) - || ((location.indexOf(':') == -1) && moduleLoadPath.startsWith(XmldbURI.XMLDB_URI_PREFIX))) { - - // Is the module source stored in the database? - try { - XmldbURI locationUri = XmldbURI.xmldbUriFor(location); + final Source moduleSource; + if (location.startsWith(XmldbURI.XMLDB_URI_PREFIX) || ((location.indexOf(':') == -1) && moduleLoadPath.startsWith(XmldbURI.XMLDB_URI_PREFIX))) { + // Load module source from database location + moduleSource = importModuleFromDb(location); + } else { + // Load module source from file or URL + moduleSource = importModuleFromContextPathAndLocation(location, namespaceURI); + } - if (moduleLoadPath.startsWith(XmldbURI.XMLDB_URI_PREFIX)) { - final XmldbURI moduleLoadPathUri = XmldbURI.xmldbUriFor(moduleLoadPath); - locationUri = moduleLoadPathUri.resolveCollectionPath(locationUri); - } + return compileOrBorrowModule(prefix, namespaceURI, location, moduleSource); + } - try (final LockedDocument lockedSourceDoc = getBroker().getXMLResource(locationUri.toCollectionPathURI(), LockMode.READ_LOCK)) { + protected Source importModuleFromDb(final String location) throws XPathException { + try { - final DocumentImpl sourceDoc = lockedSourceDoc == null ? null : lockedSourceDoc.getDocument(); - if (sourceDoc == null) { - throw moduleLoadException("Module location hint URI '" + location + "' does not refer to anything.", location); - } + XmldbURI locationUri = XmldbURI.xmldbUriFor(location); + if (moduleLoadPath.startsWith(XmldbURI.XMLDB_URI_PREFIX)) { + final XmldbURI moduleLoadPathUri = XmldbURI.xmldbUriFor(moduleLoadPath); + locationUri = moduleLoadPathUri.resolveCollectionPath(locationUri); + } - if ((sourceDoc.getResourceType() != DocumentImpl.BINARY_FILE) || !MediaType.APPLICATION_XQUERY.equals(sourceDoc.getMediaType())) { - throw moduleLoadException("Module location hint URI '" + location + "' does not refer to an XQuery.", location); - } + try (final LockedDocument lockedSourceDoc = getBroker().getXMLResource(locationUri.toCollectionPathURI(), LockMode.READ_LOCK)) { - final Source moduleSource = new DBSource(getBroker(), (BinaryDocument) sourceDoc, true); - return compileOrBorrowModule(prefix, namespaceURI, location, moduleSource); + final DocumentImpl sourceDoc = lockedSourceDoc == null ? null : lockedSourceDoc.getDocument(); + if (sourceDoc == null) { + throw moduleLoadException("Module location hint URI '" + location + "' does not refer to anything.", location); + } - } catch (final PermissionDeniedException e) { - throw moduleLoadException("Permission denied to read module source from location hint URI '" + location + ".", location, e); + if ((sourceDoc.getResourceType() != DocumentImpl.BINARY_FILE) || !MediaType.APPLICATION_XQUERY.equals(sourceDoc.getMediaType())) { + throw moduleLoadException("Module location hint URI '" + location + "' does not refer to an XQuery.", location); } - } catch (final URISyntaxException e) { - throw moduleLoadException("Invalid module location hint URI '" + location + "'.", location, e); - } - } else { + return DbUriSource.from(getBroker().getBrokerPool(), getBroker().getCurrentSubject(), sourceDoc, true, false); - // No. Load from file or URL - final Source moduleSource; - try { - //TODO: use URIs to ensure proper resolution of relative locations - moduleSource = SourceFactory.getSource(getBroker(), moduleLoadPath, location, true); - if (moduleSource == null) { - throw moduleLoadException("Source for module '" + namespaceURI + "' not found module location hint URI '" + location + "'.", location); - } - } catch (final MalformedURLException e) { - throw moduleLoadException("Invalid module location hint URI '" + location + "'.", location, e); - } catch (final IOException e) { - throw moduleLoadException("Source for module '" + namespaceURI + "' could not be read, module location hint URI '" + location + "'.", location, e); } catch (final PermissionDeniedException e) { throw moduleLoadException("Permission denied to read module source from location hint URI '" + location + ".", location, e); } + } catch (final URISyntaxException e) { + throw moduleLoadException("Invalid module location hint URI '" + location + "'.", location, e); + } + } + + protected Source importModuleFromContextPathAndLocation(final String location, final String namespaceURI) throws XPathException { + try { + final String contextPath; + if (source instanceof FileSource) { + final Path sourcePath = ((FileSource) source).getPath(); + contextPath = sourcePath.resolveSibling(moduleLoadPath).normalize().toString(); + } else { + contextPath = moduleLoadPath; + } - return compileOrBorrowModule(prefix, namespaceURI, location, moduleSource); + @Nullable final Source moduleSource = SourceFactory.getSource(getBroker(), contextPath, location, true); + if (moduleSource == null) { + throw moduleLoadException("Source for module '" + namespaceURI + "' " + "not found module location hint URI '" + location + "'.", location); + } + return moduleSource; + + } catch (final MalformedURLException e) { + throw moduleLoadException("Invalid module location hint URI '" + location + "'.", location, e); + } catch (final IOException e) { + throw moduleLoadException("Source for module '" + namespaceURI + "' could not be read, module location hint URI '" + location + "'.", location, e); + } catch (final PermissionDeniedException e) { + throw moduleLoadException("Permission denied to read module source from location hint URI '" + location + ".", location, e); } } diff --git a/exist-core/src/main/java/org/exist/xquery/XQueryUtil.java b/exist-core/src/main/java/org/exist/xquery/XQueryUtil.java index 94c14cbcde..80af5bf6b2 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQueryUtil.java +++ b/exist-core/src/main/java/org/exist/xquery/XQueryUtil.java @@ -22,7 +22,6 @@ import com.evolvedbinary.j8fu.function.BiConsumerE; import com.evolvedbinary.j8fu.function.ConsumerE; -import com.evolvedbinary.j8fu.function.Function2E; import org.exist.security.PermissionDeniedException; import org.exist.source.Source; import org.exist.storage.BrokerPool; @@ -32,12 +31,20 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.io.Writer; import java.util.Properties; +/** + * Utility to make compiling and executing XQuery simpler + * and safer by ensuring cleanup when the returned object is closed. + * + * @author Adam Retter + */ public class XQueryUtil { /** * Execute an XPath or XQuery. + * Performs both the compile and execute steps. * * @param broker the database broker. * @param source the source of the query. @@ -60,6 +67,7 @@ public static XQueryUtil.QueryResult query(final DBBroker broker, final Source s /** * Execute an XPath or XQuery. + * Performs both the compile and execute steps. * * @param broker the database broker. * @param source the source of the query. @@ -78,6 +86,26 @@ public static XQueryUtil.QueryResult query(final DBBroker broker, final Source s * @throws XPathException if the query raises an error. */ public static XQueryUtil.QueryResult query(final DBBroker broker, final Source source, final boolean isXPointer, final boolean cacheQuery, @Nullable final Sequence contextSequence, @Nullable final Properties outputProperties, @Nullable final ConsumerE preCompilationContext, @Nullable final ConsumerE preExecutionContext, @Nullable final BiConsumerE postExecutionContext) throws XPathException, PermissionDeniedException, IOException { + final CompilationResult compilationResult = compile(broker, source, isXPointer, cacheQuery, preCompilationContext); + return execute(broker, compilationResult, contextSequence, outputProperties, preExecutionContext, postExecutionContext); + } + + /** + * Compile an XPath or XQuery. + * + * @param broker the database broker. + * @param source the source of the query. + * @param isXPointer true if the query is an XPointer, false otherwise. + * @param cacheQuery true if the compiled query should be cached, false otherwise. + * @param preCompilationContext access to the XQuery Context pre-compilation, or null if access is not required. + * + * @return the result of the compilation and the compilation time. + * + * @throws IOException if the source of the query cannot be read. + * @throws PermissionDeniedException if the calling user does not have permission to execute the query. + * @throws XPathException if the query raises an error. + */ + public static CompilationResult compile(final DBBroker broker, final Source source, final boolean isXPointer, final boolean cacheQuery, @Nullable final ConsumerE preCompilationContext) throws XPathException, PermissionDeniedException, IOException { final BrokerPool brokerPool = broker.getBrokerPool(); final XQuery xquery = brokerPool.getXQueryService(); @@ -104,7 +132,7 @@ public static XQueryUtil.QueryResult query(final DBBroker broker, final Source s compiledXquery.getContext().updateContext(xqueryContext); xqueryContext.getWatchDog().reset(); - compilationTime = XQueryUtil.QueryResult.RETRIEVED_CACHED_COMPILED_QUERY; + compilationTime = XQueryUtil.CompilationResult.RETRIEVED_CACHED_COMPILED_QUERY; } else { xqueryContext = new XQueryContext(brokerPool); @@ -118,64 +146,99 @@ public static XQueryUtil.QueryResult query(final DBBroker broker, final Source s compilationTime = System.currentTimeMillis() - compilationStart; } + return new CompilationResult(source, xqueryPool, compiledXquery, xqueryContext, compilationTime); + + } catch (final XPathException | PermissionDeniedException | IOException e) { + // Make sure to clean-up in case of an exception! + if (xqueryContext != null) { + xqueryContext.runCleanupTasks(); + } + + if (xqueryPool != null && compiledXquery != null) { + xqueryPool.returnCompiledXQuery(source, compiledXquery); + } + + throw e; + } + } + + /** + * Execute a compiled XPath or XQuery. + * + * @param broker the database broker. + * @param compilationResult the compiled query. + * @param contextSequence the context sequence to use when executing the query, or null if there is no context sequence. + * @param outputProperties any output properties, or null. + * @param preExecutionContext access to the XQuery Context pre-execution, or null if access is not required. + * @param postExecutionContext access to the XQuery Context post-execution, or null if access is not required. + * + * @return the result of the query and compilation and execution times. + * + * @throws PermissionDeniedException if the calling user does not have permission to execute the query. + * @throws XPathException if the query raises an error. + */ + public static QueryResult execute(final DBBroker broker, final CompilationResult compilationResult, @Nullable final Sequence contextSequence, @Nullable final Properties outputProperties, @Nullable final ConsumerE preExecutionContext, @Nullable final BiConsumerE postExecutionContext) throws XPathException, PermissionDeniedException { + final BrokerPool brokerPool = broker.getBrokerPool(); + final XQuery xquery = brokerPool.getXQueryService(); + + try { if (preExecutionContext != null) { - preExecutionContext.accept(xqueryContext); + preExecutionContext.accept(compilationResult.xqueryContext); } final long executionStart = System.currentTimeMillis(); - final Sequence result = xquery.execute(broker, compiledXquery, null, contextSequence, outputProperties, true); + final Sequence result = xquery.execute(broker, compilationResult.compiledXquery, null, contextSequence, outputProperties, true); final long executionTime = System.currentTimeMillis() - executionStart; - final QueryResult queryResult = new XQueryUtil.QueryResult(source, xqueryPool, compiledXquery, xqueryContext, compilationTime, executionTime, result); + final QueryResult queryResult = new XQueryUtil.QueryResult(compilationResult, executionTime, result); if (postExecutionContext != null) { - postExecutionContext.accept(xqueryContext, queryResult); + postExecutionContext.accept(compilationResult.xqueryContext, queryResult); } return queryResult; - } catch (final XPathException | PermissionDeniedException | IOException e) { + } catch (final XPathException | PermissionDeniedException e) { // Make sure to clean-up in case of an exception! - if (xqueryContext != null) { - xqueryContext.runCleanupTasks(); - } + compilationResult.xqueryContext.runCleanupTasks(); - if (xqueryPool != null && compiledXquery != null) { - xqueryPool.returnCompiledXQuery(source, compiledXquery); + if (compilationResult.xqueryPool != null) { + compilationResult.xqueryPool.returnCompiledXQuery(compilationResult.source, compilationResult.compiledXquery); } throw e; } } - public static class QueryResult implements AutoCloseable { + public static class CompilationResult implements AutoCloseable { /** * Indicates that the query did not need to be compiled as a cached version was available. */ public static final int RETRIEVED_CACHED_COMPILED_QUERY = -1; - private final Source source; + public final Source source; private @Nullable final XQueryPool xqueryPool; - private @Nullable final CompiledXQuery compiledXquery; + // NOTE(AR) package-private so that we can access it from tests + final CompiledXQuery compiledXquery; private final XQueryContext xqueryContext; + private boolean closed = false; public final long compilationTime; - public final long executionTime; - public final Sequence result; - public QueryResult(final Source source, @Nullable final XQueryPool xqueryPool, @Nullable final CompiledXQuery compiledXquery, final XQueryContext xqueryContext, final long compilationTime, final long executionTime, final Sequence result) { + public CompilationResult(final Source source, @Nullable final XQueryPool xqueryPool, final CompiledXQuery compiledXquery, final XQueryContext xqueryContext, final long compilationTime) { this.source = source; this.xqueryPool = xqueryPool; this.compiledXquery = compiledXquery; this.xqueryContext = xqueryContext; - this.compilationTime = compilationTime; - this.executionTime = executionTime; - this.result = result; } @Override public void close() { + if (closed) { + return; + } + // NOTE(AR) Only when the user has finished with the Query Result i.e. {@link #result}, can we then clean-up any associated resources. xqueryContext.runCleanupTasks(); @@ -183,81 +246,92 @@ public void close() { if (xqueryPool != null && compiledXquery != null) { xqueryPool.returnCompiledXQuery(source, compiledXquery); } + + closed = true; } - } - /** - * Compile a query and perform an operation with the compiled query. - * - * @param the type of the result of the operation. - * - * @param broker the database broker. - * @param source the source of the query. - * - * @param op the operation to perform with the compiled query. - * - * @return the result of the operation. - */ - public static T withCompiledQuery(final DBBroker broker, final Source source, final Function2E op) throws XPathException, PermissionDeniedException, IOException { - final BrokerPool pool = broker.getBrokerPool(); - final XQuery xqueryService = pool.getXQueryService(); - final XQueryPool xqueryPool = pool.getXQueryPool(); - final CompiledXQuery compiledQuery = compileQuery(broker, xqueryService, xqueryPool, source); - try { - return op.apply(compiledQuery); - } finally { - if (compiledQuery != null) { - if (compiledQuery.getContext() != null) { - compiledQuery.getContext().runCleanupTasks(); - } - xqueryPool.returnCompiledXQuery(source, compiledQuery); - } + /** + * Should only be called from the XML:DB API for CompiledExpression. + * + * @param writer the output destination. + */ + public void dump(final Writer writer) { + compiledXquery.dump(writer); + } + + /** + * Should only be called from the XML:DB API for CompiledExpression. + */ + public void reset() { + compiledXquery.reset(); } } - private static CompiledXQuery compileQuery(final DBBroker broker, final XQuery xqueryService, final XQueryPool xqueryPool, final Source query) throws PermissionDeniedException, XPathException, IOException { - @Nullable CompiledXQuery compiled = null; - @Nullable XQueryContext context = null; - try { - compiled = xqueryPool.borrowCompiledXQuery(broker, query); - if (compiled == null) { - context = new XQueryContext(broker.getBrokerPool()); + public static class QueryResult implements AutoCloseable { + private final Source source; + private @Nullable final XQueryPool xqueryPool; + private @Nullable final CompiledXQuery compiledXquery; + private final XQueryContext xqueryContext; + private boolean closed = false; + + public final long compilationTime; + public final long executionTime; + public final Sequence result; + + private QueryResult(final Source source, @Nullable final XQueryPool xqueryPool, @Nullable final CompiledXQuery compiledXquery, final XQueryContext xqueryContext, final long compilationTime, final long executionTime, final Sequence result) { + this.source = source; + this.xqueryPool = xqueryPool; + this.compiledXquery = compiledXquery; + this.xqueryContext = xqueryContext; + + this.compilationTime = compilationTime; + this.executionTime = executionTime; + this.result = result; + } + + private QueryResult(final CompilationResult compilationResult, final long executionTime, final Sequence result) { + this.source = compilationResult.source; + this.xqueryPool = compilationResult.xqueryPool; + this.compiledXquery = compilationResult.compiledXquery; + this.xqueryContext = compilationResult.xqueryContext; + + this.compilationTime = compilationResult.compilationTime; + this.executionTime = executionTime; + this.result = result; + + // transfer ownership of the resources owned by compilationResult to this class + compilationResult.closed = true; + } + + /** + * Get the total time to produce the query result. + * This is compilation time (if any) plus the execution time. + * + * @return total time to produce the query result. + */ + public long totalTime() { + if (compilationTime == XQueryUtil.CompilationResult.RETRIEVED_CACHED_COMPILED_QUERY) { + return executionTime; } else { - context = compiled.getContext(); - context.prepareForReuse(); + return compilationTime + executionTime; } + } - if (compiled == null) { - compiled = xqueryService.compile(context, query); - } else { - compiled.getContext().updateContext(context); - context.getWatchDog().reset(); + @Override + public void close() { + if (closed) { + return; } - return compiled; + // NOTE(AR) Only when the user has finished with the Query Result i.e. {@link #result}, can we then clean-up any associated resources. + xqueryContext.runCleanupTasks(); - } catch (final PermissionDeniedException | XPathException | IOException e) { - if (context != null) { - context.runCleanupTasks(); - } - if (compiled != null) { - xqueryPool.returnCompiledXQuery(query, compiled); + // Once we have cleaned-up if the query should be cached we return it to the query pool. + if (xqueryPool != null && compiledXquery != null) { + xqueryPool.returnCompiledXQuery(source, compiledXquery); } - throw e; - } - } - /** - * Execute a compiled query. - * - * @param broker the database broker. - * @param compiledXQuery the compiled query. - * - * @return the result sequence. - */ - public static Sequence executeQuery(final DBBroker broker, final CompiledXQuery compiledXQuery) throws PermissionDeniedException, XPathException { - final BrokerPool pool = broker.getBrokerPool(); - final XQuery xqueryService = pool.getXQueryService(); - return xqueryService.execute(broker, compiledXQuery, null, null, null, true); + closed = true; + } } } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/util/CollectionName.java b/exist-core/src/main/java/org/exist/xquery/functions/util/CollectionName.java index e1be321c94..11a37ed664 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/util/CollectionName.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/util/CollectionName.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -36,14 +60,11 @@ import org.exist.xquery.value.FunctionParameterSequenceType; import org.exist.xquery.value.FunctionReturnSequenceType; import org.exist.xquery.value.Item; -import org.exist.xquery.value.JavaObjectValue; import org.exist.xquery.value.NodeValue; import org.exist.xquery.value.Sequence; import org.exist.xquery.value.SequenceType; import org.exist.xquery.value.StringValue; import org.exist.xquery.value.Type; -import org.xmldb.api.base.Collection; -import org.xmldb.api.base.XMLDBException; /** * @author Wolfgang Meier @@ -69,9 +90,7 @@ public CollectionName(XQueryContext context) { super(context, signature); } - /* (non-Javadoc) - * @see org.exist.xquery.BasicFunction#eval(org.exist.xquery.value.Sequence[], org.exist.xquery.value.Sequence) - */ + @Override public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathException { @@ -79,17 +98,7 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) return Sequence.EMPTY_SEQUENCE; } final Item item = args[0].itemAt(0); - if(item.getType() == Type.JAVA_OBJECT) { - final Object o = ((JavaObjectValue) item).getObject(); - if (!(o instanceof Collection)) - {throw new XPathException(this, "Passed Java object should be of type org.xmldb.api.base.Collection");} - final Collection collection = (Collection)o; - try { - return new StringValue(this, collection.getName()); - } catch (final XMLDBException e) { - throw new XPathException(this, "Failed to retrieve collection name", e); - } - } else if (Type.subTypeOf(item.getType(), Type.STRING)) { + if (Type.subTypeOf(item.getType(), Type.STRING)) { final String path = item.getStringValue(); try { final XmldbURI uri = XmldbURI.xmldbUriFor(path).removeLastSegment(); @@ -104,10 +113,9 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) //TODO: use xmldbUri return new StringValue(this, p.getOwnerDocument().getCollection().getURI().toString()); } - } else - {throw new XPathException(this, "First argument to util:collection-name should be either " + - "a Java object of type org.xmldb.api.base.Collection or a node; got: " + - Type.getTypeName(item.getType()));} + } else { + throw new XPathException(this, "First argument to util:collection-name should be either a node() or xs:string type; got: " + Type.getTypeName(item.getType())); + } return Sequence.EMPTY_SEQUENCE; } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/util/Eval.java b/exist-core/src/main/java/org/exist/xquery/functions/util/Eval.java index bd5654d822..72211150b8 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/util/Eval.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/util/Eval.java @@ -46,14 +46,14 @@ import org.apache.commons.io.output.StringBuilderWriter; import org.exist.Namespaces; -import org.exist.dom.persistent.BinaryDocument; import org.exist.dom.persistent.DocumentImpl; import org.exist.dom.persistent.DocumentSet; import org.exist.dom.memtree.NodeImpl; import org.exist.dom.memtree.SAXAdapter; import org.exist.dom.persistent.LockedDocument; import org.exist.security.PermissionDeniedException; -import org.exist.source.DBSource; +import org.exist.source.DbStoreSource; +import org.exist.source.DbUriSource; import org.exist.source.FileSource; import org.exist.source.Source; import org.exist.source.SourceFactory; @@ -65,7 +65,6 @@ import org.exist.util.serializer.XQuerySerializer; import org.exist.xmldb.XmldbURI; import org.exist.xquery.*; -import org.exist.xquery.functions.fn.FnModule; import org.exist.xquery.functions.fn.FunSerialize; import org.exist.xquery.functions.fn.FunSubSequence; import org.exist.xquery.value.*; @@ -304,8 +303,8 @@ private Sequence doEval(final XQueryContext evalContext, final Sequence contextS if (Type.subTypeOf(expr.getType(), Type.ANY_URI)) { String uri = null; - if (querySource instanceof DBSource) { - final XmldbURI documentPath = ((DBSource)querySource).getDocumentPath(); + if (querySource instanceof DbStoreSource) { + final XmldbURI documentPath = ((DbStoreSource) querySource).getDocumentPath(); uri = XmldbURI.EMBEDDED_SERVER_URI.append(documentPath).removeLastSegment().toString(); } else if (querySource instanceof FileSource) { uri = ((FileSource) querySource).getPath().getParent().toString(); @@ -535,7 +534,7 @@ private Source loadQueryFromURI(final Item expr) throws XPathException, NullPoin throw new XPathException(this, "source for module " + location + " is not an XQuery or " + "declares a wrong mime-type"); } - querySource = new DBSource(context.getBroker(), (BinaryDocument) sourceDoc, true); + querySource = DbUriSource.from(context.getBroker().getBrokerPool(), context.getBroker().getCurrentSubject(), sourceDoc, true, false); } catch (final PermissionDeniedException e) { throw new XPathException(this, "permission denied to read module source from " + location); } diff --git a/exist-core/src/main/resources/org/exist/validation/internal/query/find_catalogs_with_dtd.xq b/exist-core/src/main/resources/org/exist/validation/internal/query/find_catalogs_with_dtd.xq index 06e876aaaf..dcdc137311 100644 --- a/exist-core/src/main/resources/org/exist/validation/internal/query/find_catalogs_with_dtd.xq +++ b/exist-core/src/main/resources/org/exist/validation/internal/query/find_catalogs_with_dtd.xq @@ -52,8 +52,6 @@ Returns: Sequence of document-uris. - - $Id$ :) declare namespace ctlg = "urn:oasis:names:tc:entity:xmlns:xml:catalog"; diff --git a/exist-core/src/main/resources/org/exist/validation/internal/query/find_schema_by_targetNamespace.xq b/exist-core/src/main/resources/org/exist/validation/internal/query/find_schema_by_targetNamespace.xq index 84cda306bd..f472fcebc5 100644 --- a/exist-core/src/main/resources/org/exist/validation/internal/query/find_schema_by_targetNamespace.xq +++ b/exist-core/src/main/resources/org/exist/validation/internal/query/find_schema_by_targetNamespace.xq @@ -52,8 +52,6 @@ Returns: Sequence of document-uris. - - $Id$ :) declare variable $local:collection as xs:string external; diff --git a/exist-core/src/test/java/org/exist/source/SourceFactoryTest.java b/exist-core/src/test/java/org/exist/source/SourceFactoryTest.java index 769429ef1e..0a7464860c 100644 --- a/exist-core/src/test/java/org/exist/source/SourceFactoryTest.java +++ b/exist-core/src/test/java/org/exist/source/SourceFactoryTest.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -23,10 +47,14 @@ import com.googlecode.junittoolbox.ParallelRunner; +import org.exist.EXistException; import org.exist.dom.persistent.BinaryDocument; import org.exist.dom.persistent.LockedDocument; import org.exist.security.PermissionDeniedException; +import org.exist.storage.BrokerPool; import org.exist.storage.DBBroker; +import org.exist.storage.txn.TransactionManager; +import org.exist.storage.txn.Txn; import org.exist.xmldb.XmldbURI; import org.junit.Test; import org.junit.runner.RunWith; @@ -35,6 +63,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Paths; +import java.util.Optional; import static org.easymock.EasyMock.*; import static org.junit.Assert.*; @@ -200,243 +229,345 @@ public void getSourceFromResource_contextFolderUrl_locationRelativeUrl_basedOnSo } @Test - public void getSourceFromXmldb_noContext() throws IOException, PermissionDeniedException { + public void getSourceFromXmldb_noContext() throws IOException, PermissionDeniedException, EXistException { final String contextPath = null; final String location = "xmldb:exist:///db/library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); final LockedDocument mockLockedDoc = createMock(LockedDocument.class); final BinaryDocument mockBinDoc = createMock(BinaryDocument.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(mockLockedDoc); expect(mockLockedDoc.getDocument()).andReturn(mockBinDoc); - expect(mockBinDoc.getResourceType()).andReturn(BinaryDocument.BINARY_FILE); - expect(mockBinDoc.getURI()).andReturn(XmldbURI.create(location)).times(2); expect(mockBinDoc.getLastModified()).andReturn(123456789l); + /*expect*/ mockTxn.commit(); /*expect*/ mockLockedDoc.close(); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker, mockLockedDoc, mockBinDoc); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); - assertTrue(libSource instanceof DBSource); - assertEquals(XmldbURI.create(location), ((DBSource)libSource).getDocumentPath()); + assertTrue(libSource instanceof DbUriSource); + assertEquals(XmldbURI.create(location), ((DbUriSource)libSource).getDocumentPath()); - verify(mockBroker, mockLockedDoc, mockBinDoc); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); } @Test - public void getSourceFromXmldb() throws IOException, PermissionDeniedException { + public void getSourceFromXmldb() throws IOException, PermissionDeniedException, EXistException { final String contextPath = "xmldb:exist:///db"; final String location = "library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); final LockedDocument mockLockedDoc = createMock(LockedDocument.class); final BinaryDocument mockBinDoc = createMock(BinaryDocument.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(mockLockedDoc); expect(mockLockedDoc.getDocument()).andReturn(mockBinDoc); - expect(mockBinDoc.getResourceType()).andReturn(BinaryDocument.BINARY_FILE); - expect(mockBinDoc.getURI()).andReturn(XmldbURI.create(contextPath).append(location)).times(2); expect(mockBinDoc.getLastModified()).andReturn(123456789l); + /*expect*/ mockTxn.commit(); /*expect*/ mockLockedDoc.close(); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker, mockLockedDoc, mockBinDoc); + replay(mockBrokerPool, mockBroker,mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); - assertTrue(libSource instanceof DBSource); - assertEquals(XmldbURI.create(contextPath).append(location), ((DBSource)libSource).getDocumentPath()); + assertTrue(libSource instanceof DbUriSource); + assertEquals(XmldbURI.create(contextPath).append(location), ((DbUriSource)libSource).getDocumentPath()); - verify(mockBroker, mockLockedDoc, mockBinDoc); + verify(mockBrokerPool, mockBroker,mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); } @Test - public void getNonExistentSourceFromXmldb_noContext() throws IOException, PermissionDeniedException { + public void getNonExistentSourceFromXmldb_noContext() throws IOException, PermissionDeniedException, EXistException { final String contextPath = null; final String location = "xmldb:exist:///db/library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(null); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); assertNull(libSource); - verify(mockBroker); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); } @Test - public void getNonExistentSourceFromXmldb() throws IOException, PermissionDeniedException { + public void getNonExistentSourceFromXmldb() throws IOException, PermissionDeniedException, EXistException { final String contextPath = "xmldb:exist:///db"; final String location = "library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(null); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); assertNull(libSource); - verify(mockBroker); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); } @Test - public void getSourceFromXmldbEmbedded_noContext() throws IOException, PermissionDeniedException { + public void getSourceFromXmldbEmbedded_noContext() throws IOException, PermissionDeniedException, EXistException { final String contextPath = null; final String location = "xmldb:exist://embedded-eXist-server/db/library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); final LockedDocument mockLockedDoc = createMock(LockedDocument.class); final BinaryDocument mockBinDoc = createMock(BinaryDocument.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(mockLockedDoc); expect(mockLockedDoc.getDocument()).andReturn(mockBinDoc); - expect(mockBinDoc.getResourceType()).andReturn(BinaryDocument.BINARY_FILE); - expect(mockBinDoc.getURI()).andReturn(XmldbURI.create(location)).times(2); expect(mockBinDoc.getLastModified()).andReturn(123456789l); + /*expect*/ mockTxn.commit(); /*expect*/ mockLockedDoc.close(); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker, mockLockedDoc, mockBinDoc); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); - assertTrue(libSource instanceof DBSource); - assertEquals(XmldbURI.create(location), ((DBSource)libSource).getDocumentPath()); + assertTrue(libSource instanceof DbUriSource); + assertEquals(XmldbURI.create(location), ((DbUriSource)libSource).getDocumentPath()); - verify(mockBroker, mockLockedDoc, mockBinDoc); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); } @Test - public void getSourceFromXmldbEmbedded() throws IOException, PermissionDeniedException { + public void getSourceFromXmldbEmbedded() throws IOException, PermissionDeniedException, EXistException { final String contextPath = "xmldb:exist://embedded-eXist-server/db"; final String location = "library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); final LockedDocument mockLockedDoc = createMock(LockedDocument.class); final BinaryDocument mockBinDoc = createMock(BinaryDocument.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(mockLockedDoc); expect(mockLockedDoc.getDocument()).andReturn(mockBinDoc); - expect(mockBinDoc.getResourceType()).andReturn(BinaryDocument.BINARY_FILE); - expect(mockBinDoc.getURI()).andReturn(XmldbURI.create(contextPath).append(location)).times(2); expect(mockBinDoc.getLastModified()).andReturn(123456789l); + /*expect*/ mockTxn.commit(); /*expect*/ mockLockedDoc.close(); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker, mockLockedDoc, mockBinDoc); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); - assertTrue(libSource instanceof DBSource); - assertEquals(XmldbURI.create(contextPath).append(location), ((DBSource)libSource).getDocumentPath()); + assertTrue(libSource instanceof DbUriSource); + assertEquals(XmldbURI.create(contextPath).append(location), ((DbUriSource)libSource).getDocumentPath()); - verify(mockBroker, mockLockedDoc, mockBinDoc); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); } @Test - public void getNonExistentSourceFromXmldbEmbedded_noContext() throws IOException, PermissionDeniedException { + public void getNonExistentSourceFromXmldbEmbedded_noContext() throws IOException, PermissionDeniedException, EXistException { final String contextPath = null; final String location = "xmldb:exist://embedded-eXist-server/db/library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(null); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); assertNull(libSource); - verify(mockBroker); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); } @Test - public void getNonExistentSourceFromXmldbEmbedded() throws IOException, PermissionDeniedException { + public void getNonExistentSourceFromXmldbEmbedded() throws IOException, PermissionDeniedException, EXistException { final String contextPath = "xmldb:exist://embedded-eXist-server/db"; final String location = "library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(null); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); assertNull(libSource); - verify(mockBroker); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); } @Test - public void getSourceFromDb() throws IOException, PermissionDeniedException { + public void getSourceFromDb() throws IOException, PermissionDeniedException, EXistException { final String contextPath = "/db"; final String location = "library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); final LockedDocument mockLockedDoc = createMock(LockedDocument.class); final BinaryDocument mockBinDoc = createMock(BinaryDocument.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(mockLockedDoc); expect(mockLockedDoc.getDocument()).andReturn(mockBinDoc); - expect(mockBinDoc.getResourceType()).andReturn(BinaryDocument.BINARY_FILE); - expect(mockBinDoc.getURI()).andReturn(XmldbURI.create(contextPath).append(location)).times(2); expect(mockBinDoc.getLastModified()).andReturn(123456789l); + /*expect*/ mockTxn.commit(); /*expect*/ mockLockedDoc.close(); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker, mockLockedDoc, mockBinDoc); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); - assertTrue(libSource instanceof DBSource); - assertEquals(XmldbURI.create(contextPath).append(location), ((DBSource)libSource).getDocumentPath()); + assertTrue(libSource instanceof DbUriSource); + assertEquals(XmldbURI.create(contextPath).append(location), ((DbUriSource)libSource).getDocumentPath()); - verify(mockBroker, mockLockedDoc, mockBinDoc); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); } @Test - public void getSourceFromDb_noContext() throws IOException, PermissionDeniedException { + public void getSourceFromDb_noContext() throws IOException, PermissionDeniedException, EXistException { final String contextPath = null; final String location = "/db/library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); final LockedDocument mockLockedDoc = createMock(LockedDocument.class); final BinaryDocument mockBinDoc = createMock(BinaryDocument.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(mockLockedDoc); expect(mockLockedDoc.getDocument()).andReturn(mockBinDoc); - expect(mockBinDoc.getResourceType()).andReturn(BinaryDocument.BINARY_FILE); - expect(mockBinDoc.getURI()).andReturn(XmldbURI.create(location)).times(2); expect(mockBinDoc.getLastModified()).andReturn(123456789l); + /*expect*/ mockTxn.commit(); /*expect*/ mockLockedDoc.close(); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker, mockLockedDoc, mockBinDoc); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); - assertTrue(libSource instanceof DBSource); - assertEquals(XmldbURI.create(location), ((DBSource)libSource).getDocumentPath()); + assertTrue(libSource instanceof DbUriSource); + assertEquals(XmldbURI.create(location), ((DbUriSource)libSource).getDocumentPath()); - verify(mockBroker, mockLockedDoc, mockBinDoc); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn, mockLockedDoc, mockBinDoc); } @Test - public void getNonExistentSourceFromDb() throws IOException, PermissionDeniedException { + public void getNonExistentSourceFromDb() throws IOException, PermissionDeniedException, EXistException { final String contextPath = "/db"; final String location = "library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(null); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); assertNull(libSource); - verify(mockBroker); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); } @Test - public void getNonExistentSourceFromDb_noContext() throws IOException, PermissionDeniedException { + public void getNonExistentSourceFromDb_noContext() throws IOException, PermissionDeniedException, EXistException { final String contextPath = null; final String location = "/db/library.xqm"; + final BrokerPool mockBrokerPool = createMock(BrokerPool.class); final DBBroker mockBroker = createMock(DBBroker.class); + final TransactionManager mockTransactionManager = createMock(TransactionManager.class); + final Txn mockTxn = createMock(Txn.class); + expect(mockBroker.getBrokerPool()).andReturn(mockBrokerPool); + expect(mockBrokerPool.get(Optional.empty())).andReturn(mockBroker); + expect(mockBrokerPool.getTransactionManager()).andReturn(mockTransactionManager); + expect(mockTransactionManager.beginTransaction()).andReturn(mockTxn); expect(mockBroker.getXMLResource(anyObject(), anyObject())).andReturn(null); + /*expect*/ mockTxn.close(); + /*expect*/ mockBroker.close(); - replay(mockBroker); + replay(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); final Source libSource = SourceFactory.getSource(mockBroker, contextPath, location, false); assertNull(libSource); - verify(mockBroker); + verify(mockBrokerPool, mockBroker, mockTransactionManager, mockTxn); } @Test diff --git a/exist-core/src/test/java/org/exist/storage/ConcurrentBrokerPoolTest.java b/exist-core/src/test/java/org/exist/storage/ConcurrentBrokerPoolTest.java index 0af9b7aa42..471e0ff794 100644 --- a/exist-core/src/test/java/org/exist/storage/ConcurrentBrokerPoolTest.java +++ b/exist-core/src/test/java/org/exist/storage/ConcurrentBrokerPoolTest.java @@ -187,7 +187,7 @@ public Tuple2 call() throws Exception { server.startDb(); try { store(server.getBrokerPool()); - return Tuple(server.getTemporaryStorage().get(), uuid); + return Tuple(server.getTemporaryStorage(), uuid); } finally { server.stopDb(false); // NOTE: false flag ensures we don't delete the temporary storage! } diff --git a/exist-core/src/test/java/org/exist/xquery/CleanupTest.java b/exist-core/src/test/java/org/exist/xquery/CleanupTest.java index 5850329f80..b1881e4a2c 100644 --- a/exist-core/src/test/java/org/exist/xquery/CleanupTest.java +++ b/exist-core/src/test/java/org/exist/xquery/CleanupTest.java @@ -56,6 +56,7 @@ import org.exist.xmldb.EXistResource; import org.exist.xmldb.EXistResourceSet; import org.exist.xmldb.EXistXQueryService; +import org.exist.xmldb.LocalCompiledExpression; import org.exist.xquery.value.FunctionReference; import org.exist.xquery.value.Sequence; import org.junit.After; @@ -63,7 +64,6 @@ import org.junit.ClassRule; import org.junit.Test; import org.xmldb.api.base.Collection; -import org.xmldb.api.base.CompiledExpression; import org.xmldb.api.base.XMLDBException; import org.xmldb.api.modules.BinaryResource; import org.xmldb.api.modules.CollectionManagementService; @@ -139,16 +139,20 @@ public void tearDown() throws XMLDBException { @Test public void resetStateOfModuleVars() throws XMLDBException, XPathException { final EXistXQueryService service = (EXistXQueryService)collection.getService("XQueryService", "1.0"); - final CompiledExpression compiled = service.compile(TEST_QUERY); - final Module[] modules = ((PathExpr) compiled).getContext().getModules(MODULE_NS); + final LocalCompiledExpression compiledExpression = (LocalCompiledExpression) service.compile(TEST_QUERY); + final XQueryUtil.CompilationResult compilationResult = compiledExpression.getCompilationResult(); + final CompiledXQuery compiledXquery = compilationResult.compiledXquery; + + + final Module[] modules = compiledXquery.getContext().getModules(MODULE_NS); assertEquals(1, modules.length); final Module module = modules[0]; final java.util.Collection varDecls = ((ExternalModule) module).getVariableDeclarations(); final Iterator vi = varDecls.iterator(); final VariableDeclaration var1 = vi.next(); final VariableDeclaration var2 = vi.next(); - final FunctionCall root = (FunctionCall) ((PathExpr) compiled).getFirst(); + final FunctionCall root = (FunctionCall) ((PathExpr) compiledXquery).getFirst(); final UserDefinedFunction calledFunc = root.getFunction(); final Expression calledBody = calledFunc.getFunctionBody(); @@ -159,7 +163,7 @@ public void resetStateOfModuleVars() throws XMLDBException, XPathException { var2.setContextDocSet(DocumentSet.EMPTY_DOCUMENT_SET); // execute query and check result - try (final EXistResourceSet result = (EXistResourceSet) service.execute(compiled)) { + try (final EXistResourceSet result = (EXistResourceSet) service.execute(compiledExpression)) { assertEquals(result.getSize(), 1); try (final EXistResource resource = (EXistResource) result.getResource(0)) { assertEquals(resource.getContent(), "Hello world123"); @@ -194,13 +198,17 @@ public void preserveExternalVariable() throws XMLDBException, XPathException { // see https://github.com/eXist-db/exist/pull/1512 and use of util:eval final EXistXQueryService service = (EXistXQueryService)collection.getService("XQueryService", "1.0"); - final CompiledExpression compiled = service.compile(INTERNAL_MODULE_EVAL_TEST); - final Module[] modules = ((PathExpr) compiled).getContext().getModules(MODULE_NS); + final LocalCompiledExpression compiledExpression = (LocalCompiledExpression) service.compile(INTERNAL_MODULE_EVAL_TEST); + final XQueryUtil.CompilationResult compilationResult = compiledExpression.getCompilationResult(); + final CompiledXQuery compiledXquery = compilationResult.compiledXquery; + + + final Module[] modules = compiledXquery.getContext().getModules(MODULE_NS); assertEquals(1, modules.length); final Module module = modules[0]; module.declareVariable(new QName("VAR", MODULE_NS, "t"), "TEST"); - try (final EXistResourceSet result = (EXistResourceSet) service.execute(compiled)) { + try (final EXistResourceSet result = (EXistResourceSet) service.execute(compiledExpression)) { assertEquals(result.getSize(), 2); try (final EXistResource resource = (EXistResource) result.getResource(1)) { assertEquals(resource.getContent(), "TEST"); @@ -215,15 +223,18 @@ public void preserveExternalVariable() throws XMLDBException, XPathException { public void resetStateofInternalModule() throws XMLDBException, XPathException { final EXistXQueryService service = (EXistXQueryService)collection.getService("XQueryService", "1.0"); - final CompiledExpression compiled = service.compile(INTERNAL_MODULE_TEST); - final Module[] modules = ((PathExpr) compiled).getContext().getModules(MODULE_NS); + final LocalCompiledExpression compiledExpression = (LocalCompiledExpression) service.compile(INTERNAL_MODULE_TEST); + final XQueryUtil.CompilationResult compilationResult = compiledExpression.getCompilationResult(); + final CompiledXQuery compiledXquery = compilationResult.compiledXquery; + + final Module[] modules = compiledXquery.getContext().getModules(MODULE_NS); assertEquals(1, modules.length); final Module module = modules[0]; module.declareVariable(new QName("VAR", MODULE_NS, "t"), "TEST"); - final InternalFunctionCall root = (InternalFunctionCall) ((PathExpr) compiled).getFirst(); + final InternalFunctionCall root = (InternalFunctionCall) ((PathExpr) compiledXquery).getFirst(); final TestModule.TestFunction func = (TestModule.TestFunction) root.getFunction(); - try (final EXistResourceSet result = (EXistResourceSet) service.execute(compiled)) { + try (final EXistResourceSet result = (EXistResourceSet) service.execute(compiledExpression)) { assertEquals(result.getSize(), 1); try (final EXistResource resource = (EXistResource) result.getResource(0)) { assertEquals(resource.getContent(), "TEST"); diff --git a/exist-core/src/test/java/org/exist/xquery/RunningXQueryRestTest.java b/exist-core/src/test/java/org/exist/xquery/RunningXQueryRestTest.java new file mode 100644 index 0000000000..7e395626d6 --- /dev/null +++ b/exist-core/src/test/java/org/exist/xquery/RunningXQueryRestTest.java @@ -0,0 +1,309 @@ +/* + * Copyright (C) 2014, Evolved Binary Ltd + * + * This file was originally ported from FusionDB to Elemental by + * Evolved Binary, for the benefit of the Elemental Open Source community. + * Only the ported code as it appears in this file, at the time that + * it was contributed to Elemental, was re-licensed under The GNU + * Lesser General Public License v2.1 only for use in Elemental. + * + * This license grant applies only to a snapshot of the code as it + * appeared when ported, it does not offer or infer any rights to either + * updates of this source code or access to the original source code. + * + * The GNU Lesser General Public License v2.1 only license follows. + * + * ===================================================================== + * + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.client.fluent.Executor; +import org.apache.http.client.fluent.Request; +import org.exist.TestUtils; +import org.exist.storage.BrokerPool; +import org.exist.test.ExistWebServer; +import org.exist.xmldb.XmldbURI; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import xyz.elemental.mediatype.MediaType; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.http.HttpStatus.SC_CREATED; +import static org.apache.http.HttpStatus.SC_OK; +import static org.exist.util.PropertiesBuilder.propertiesBuilder; +import static org.junit.Assert.*; + +/** + * Performs a number of tests against the read-locked source document of an XQuery that is currently running over the REST API. + * + * @author Adam Retter + */ +public class RunningXQueryRestTest { + + private static final XmldbURI TEST_COLLECTION_URI = XmldbURI.DB.append("running-xquery-test"); + private static final XmldbURI LATCHED_QUERY_URI = XmldbURI.create("latched-query.xq"); + private static final String RUNNING_LATCH_REF_XQUERY_VARIABLE_NAME = "query-running-latch-ref"; + private static final String EXIT_LATCH_REF_XQUERY_VARIABLE_NAME = "query-exit-latch-ref"; + + private static final long TIMEOUT_MS = 3000; + + @Rule + public final ExistWebServer existWebServer = new ExistWebServer(true, false, + propertiesBuilder() + .put(FunctionFactory.PROPERTY_ENABLE_JAVA_BINDING, true) + .put(BrokerPool.PROPERTY_SHUTDOWN_DELAY, 100L) + .build(), + true, true, false); + + @Nullable private Executor executor = null; + @Nullable private String uri = null; + @Nullable private String docUri = null; + @Nullable private String latchedQueryLastModified = null; + @Nullable private ExecutorService executorService = null; + @Nullable private Future queryResultFuture = null; + + private final String query = + "declare namespace system = 'http://exist-db.org/xquery/system';\n" + + "declare namespace lws = 'java:" + LatchWrappersSingleton.class.getName() + "';\n" + + "declare namespace at = 'java:" + AtomicReference.class.getName() + "';\n" + + "declare namespace cl = 'java:" + CountDownLatch.class.getName() + "';\n" + + "declare namespace tu = 'java:" + TimeUnit.class.getName() + "';\n" + + "declare variable $lwsi := lws:INSTANCE();\n" + + "declare variable $" + RUNNING_LATCH_REF_XQUERY_VARIABLE_NAME + " := lws:queryRunningLatchRef($lwsi);\n" + + "declare variable $" + EXIT_LATCH_REF_XQUERY_VARIABLE_NAME + " := lws:queryExitLatchRef($lwsi);\n" + + "\n" + + "let $running-latch := at:get($" + RUNNING_LATCH_REF_XQUERY_VARIABLE_NAME + ")\n" + + "let $_ := cl:countDown($running-latch)\n" + + "let $start := util:system-time()\n" + + "let $exit-latch := at:get($" + EXIT_LATCH_REF_XQUERY_VARIABLE_NAME + ")\n" + + "let $exit-latch-zero := cl:await($exit-latch, " + TIMEOUT_MS + " cast as xs:long, tu:MILLISECONDS())\n" + + "let $end := util:system-time()\n" + + "return\n" + + " {$end - $start}"; + + /** + * This singleton allows us to share access to the latches between this test in Java and the XQuery we are executing. + */ + public static class LatchWrappersSingleton { + public static final LatchWrappersSingleton INSTANCE = new LatchWrappersSingleton(); + + final AtomicReference queryRunningLatchRef = new AtomicReference<>(); + final AtomicReference queryExitLatchRef = new AtomicReference<>(); + + private LatchWrappersSingleton() { + } + } + + @Before + public void runXQuery() throws IOException, InterruptedException { + this.executor = Executor + .newInstance() + .auth(TestUtils.ADMIN_DB_USER, TestUtils.ADMIN_DB_PWD) + .authPreemptive(new HttpHost("localhost", existWebServer.getPort())); + + this.uri = "http://localhost:" + existWebServer.getPort() + "/exist/rest" + TEST_COLLECTION_URI.getCollectionPath(); + this.docUri = uri + LATCHED_QUERY_URI.getCollectionPath(); + + // Store our XQuery as document into the database + final HttpResponse storeResponse = executor.execute( + Request + .Put(docUri) + .addHeader("Content-Type", MediaType.APPLICATION_XQUERY) + .bodyByteArray(query.getBytes(UTF_8)) + ).returnResponse(); + assertEquals(SC_CREATED, storeResponse.getStatusLine().getStatusCode()); + + // Record the last modified time of the XQuery in the database + final HttpResponse headResponse = executor.execute( + Request + .Head(docUri) + .addHeader("Accept", MediaType.APPLICATION_XML) + ).returnResponse(); + assertEquals(SC_OK, headResponse.getStatusLine().getStatusCode()); + this.latchedQueryLastModified = headResponse.getFirstHeader("Last-Modified").getValue(); + + // Create a latch that we can receive a signal from when the query is running + LatchWrappersSingleton.INSTANCE.queryRunningLatchRef.set(new CountDownLatch(1)); + + // Create a latch that we can send a signal to allow the query to finish + LatchWrappersSingleton.INSTANCE.queryExitLatchRef.set(new CountDownLatch(1)); + + // Prepare a Callable that will compile and execute the XQuery (which will wait on the queryExitLatch) + final Callable latchedQueryCallable = () -> { + + final HttpResponse getResponse = executor.execute( + Request + .Get(docUri) + .addHeader("Accept", MediaType.APPLICATION_XML) + ).returnResponse(); + assertEquals(SC_OK, getResponse.getStatusLine().getStatusCode()); + + try (final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { + getResponse.getEntity().writeTo(baos); + return baos.toByteArray(); + } catch (final Exception e) { + e.printStackTrace(); + throw e; + } + }; + + // Run the latched query callable from a separate thread so that we don't block our test + this.executorService = Executors.newFixedThreadPool(2); // We size to 2 threads so that we have one for the latched query, and one for the test method itself (see tests below) + this.queryResultFuture = executorService.submit(latchedQueryCallable); + + // Wait for the query to start running + final CountDownLatch latch = LatchWrappersSingleton.INSTANCE.queryRunningLatchRef.get(); + try { + final boolean running = latch.await(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertTrue("Timeout exceeded whilst waiting for query to notify us it is running", running); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); // Restore interrupted flag + throw e; + } + } + + @After + public void stopXQuery() { + @Nullable final CountDownLatch runningLatch = LatchWrappersSingleton.INSTANCE.queryRunningLatchRef.getAndSet(null); + if (runningLatch != null) { + while (runningLatch.getCount() > 0) { + runningLatch.countDown(); + } + } + + @Nullable final CountDownLatch exitLatch = LatchWrappersSingleton.INSTANCE.queryExitLatchRef.getAndSet(null); + if (exitLatch != null) { + while (exitLatch.getCount() > 0) { + exitLatch.countDown(); + } + } + + this.queryResultFuture.cancel(true); + this.queryResultFuture = null; + this.executorService.shutdownNow(); + this.executorService = null; + this.latchedQueryLastModified = null; + this.docUri = null; + this.uri = null; + this.executor = null; + } + + /** + * When this test is run, the /db/running-xquery-test/latched-query.xq XQuery + * is already executing (see {@link #runXQuery()}) in a separate thread that holds a READ_LOCK on its Source document. + * + * We try and read the XQuery source document under those conditions, and the read should succeed. + */ + @Test + public void readRunningXQuerySourceDocument() throws InterruptedException, ExecutionException, TimeoutException { + // Prepare a Callable that will try and read the XQuery's source document + final Callable readDocumentCallable = () -> { + + // try and read the XQuery source document + try { + final HttpResponse headResponse = executor.execute( + Request + .Head(docUri) + .addHeader("Accept", MediaType.APPLICATION_XML) + ).returnResponse(); + assertEquals(SC_OK, headResponse.getStatusLine().getStatusCode()); + + return headResponse.getFirstHeader("Last-Modified").getValue(); + + } finally { + + // Instruct the running XQuery to finish + @Nullable final CountDownLatch exitLatch = LatchWrappersSingleton.INSTANCE.queryExitLatchRef.get(); + assertNotNull("exitLatch should not be null at this point", exitLatch); + exitLatch.countDown(); + } + }; + + // Run the readDocumentCallable from a separate thread so that we don't block our test + final Future documentLastModifiedFuture = executorService.submit(readDocumentCallable); + final String documentLastModified = documentLastModifiedFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertEquals(latchedQueryLastModified, documentLastModified); + + // Check the results of the query + final byte[] queryResult = queryResultFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertNotNull(queryResult); + final String queryResultStr = new String(queryResult, UTF_8); + assertTrue(queryResultStr.startsWith("PT")); + assertTrue(queryResultStr.endsWith("")); + } + + /** + * When this test is run, the /db/running-xquery-test/latched-query.xq XQuery + * is already executing (see {@link #runXQuery()}) in a separate thread, + * that should have released the READ_LOCK on its Source document after query compilation. + * + * We try and replace the XQuery source document under those conditions, which should be possible + * as no lock is held on the source document after compilation. + */ + @Test + public void replaceRunningXQuerySourceDocument() throws InterruptedException, ExecutionException, TimeoutException { + // Prepare a Callable that will try and replace the XQuery's source document + final Callable replaceDocumentCallable = () -> { + + try { + final String replacementQuery = ""; + + final HttpResponse storeResponse = executor.execute( + Request + .Put(docUri) + .addHeader("Content-Type", MediaType.APPLICATION_XQUERY) + .bodyByteArray(replacementQuery.getBytes(UTF_8)) + ).returnResponse(); + + return storeResponse.getStatusLine().getStatusCode(); + + } finally { + // Instruct the running XQuery to finish + @Nullable final CountDownLatch exitLatch = LatchWrappersSingleton.INSTANCE.queryExitLatchRef.get(); + assertNotNull("exitLatch should not be null at this point", exitLatch); + exitLatch.countDown(); + } + }; + + // Run the replaceDocumentCallable from a separate thread so that we don't block our test + final Future replacedDocumentFuture = executorService.submit(replaceDocumentCallable); + final Integer replaceDocumentStatus = replacedDocumentFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertEquals(SC_CREATED, replaceDocumentStatus.intValue()); + + // Check the results of the query + final byte[] queryResult = queryResultFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertNotNull(queryResult); + final String queryResultStr = new String(queryResult, UTF_8); + assertTrue(queryResultStr.startsWith("PT")); + assertTrue(queryResultStr.endsWith("")); + } +} diff --git a/exist-core/src/test/java/org/exist/xquery/RunningXQueryTest.java b/exist-core/src/test/java/org/exist/xquery/RunningXQueryTest.java new file mode 100644 index 0000000000..5c4a8f4bb8 --- /dev/null +++ b/exist-core/src/test/java/org/exist/xquery/RunningXQueryTest.java @@ -0,0 +1,331 @@ +/* + * Copyright (C) 2014, Evolved Binary Ltd + * + * This file was originally ported from FusionDB to Elemental by + * Evolved Binary, for the benefit of the Elemental Open Source community. + * Only the ported code as it appears in this file, at the time that + * it was contributed to Elemental, was re-licensed under The GNU + * Lesser General Public License v2.1 only for use in Elemental. + * + * This license grant applies only to a snapshot of the code as it + * appeared when ported, it does not offer or infer any rights to either + * updates of this source code or access to the original source code. + * + * The GNU Lesser General Public License v2.1 only license follows. + * + * ===================================================================== + * + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.EXistException; +import org.exist.collections.Collection; +import org.exist.dom.QName; +import org.exist.dom.persistent.BinaryDocument; +import org.exist.dom.persistent.DocumentImpl; +import org.exist.dom.persistent.LockedDocument; +import org.exist.security.PermissionDeniedException; +import org.exist.source.DbUriSource; +import org.exist.source.Source; +import org.exist.storage.BrokerPool; +import org.exist.storage.DBBroker; +import org.exist.storage.blob.BlobId; +import org.exist.storage.lock.Lock; +import org.exist.storage.lock.LockManager; +import org.exist.storage.txn.Txn; +import org.exist.test.ExistEmbeddedServer; +import org.exist.util.LockException; +import org.exist.util.StringInputSource; +import org.exist.xmldb.XmldbURI; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.Type; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; +import xyz.elemental.mediatype.MediaType; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; + +import static org.exist.util.PropertiesBuilder.propertiesBuilder; +import static org.exist.util.io.InputStreamUtil.readAll; +import static org.junit.Assert.*; + +/** + * Performs a number of tests against the read-locked source document of an XQuery that is currently running. + * + * @author Adam Retter + */ +public class RunningXQueryTest { + + private static final XmldbURI TEST_COLLECTION_URI = XmldbURI.DB.append("running-xquery-test"); + private static final XmldbURI LATCHED_QUERY_URI = XmldbURI.create("latched-query.xq"); + private static final String RUNNING_LATCH_REF_XQUERY_VARIABLE_NAME = "query-running-latch-ref"; + private static final String EXIT_LATCH_REF_XQUERY_VARIABLE_NAME = "query-exit-latch-ref"; + + private static final long TIMEOUT_MS = 3000; + + @Rule + public final ExistEmbeddedServer existEmbeddedServer = new ExistEmbeddedServer( + propertiesBuilder() + .put(FunctionFactory.PROPERTY_ENABLE_JAVA_BINDING, true) + .put(BrokerPool.PROPERTY_SHUTDOWN_DELAY, 100L) + .build(), + true, true); + + @Nullable private ExecutorService executorService = null; + private final AtomicReference lockedQuerySourceDocumentRef = new AtomicReference<>(); + private final AtomicReference queryRunningLatchRef = new AtomicReference<>(); + private final AtomicReference queryExitLatchRef = new AtomicReference<>(); + @Nullable private Future queryResultFuture = null; + + private final String query = + "declare namespace system = 'http://exist-db.org/xquery/system';\n" + + "declare namespace at = 'java:" + AtomicReference.class.getName() + "';\n" + + "declare namespace cl = 'java:" + CountDownLatch.class.getName() + "';\n" + + "declare namespace tu = 'java:" + TimeUnit.class.getName() + "';\n" + + "declare variable $" + RUNNING_LATCH_REF_XQUERY_VARIABLE_NAME + " external;\n" + + "declare variable $" + EXIT_LATCH_REF_XQUERY_VARIABLE_NAME + " external;\n" + + "\n" + + "let $running-latch := at:get($" + RUNNING_LATCH_REF_XQUERY_VARIABLE_NAME + ")\n" + + "let $_ := cl:countDown($running-latch)\n" + + "let $start := util:system-time()\n" + + "let $exit-latch := at:get($" + EXIT_LATCH_REF_XQUERY_VARIABLE_NAME + ")\n" + + "let $exit-latch-zero := cl:await($exit-latch, " + TIMEOUT_MS + " cast as xs:long, tu:MILLISECONDS())\n" + + "let $end := util:system-time()\n" + + "return\n" + + " {$end - $start}"; + + @Before + public void runXQuery() throws PermissionDeniedException, EXistException, IOException, SAXException, LockException, InterruptedException { + final BrokerPool brokerPool = existEmbeddedServer.getBrokerPool(); + + // Store our XQuery as document into the database + try (final DBBroker broker = brokerPool.get(Optional.of(brokerPool.getSecurityManager().getSystemSubject())); + final Txn transaction = brokerPool.getTransactionManager().beginTransaction(); + final Collection testCollection = broker.getOrCreateCollection(transaction, TEST_COLLECTION_URI)) { + + final MediaType xqueryMediaType = broker.getBrokerPool().getMediaTypeService().getMediaTypeResolver().fromString(MediaType.APPLICATION_XQUERY); + broker.storeDocument(transaction, LATCHED_QUERY_URI, new StringInputSource(query.getBytes(StandardCharsets.UTF_8)), xqueryMediaType, testCollection); + + transaction.commit(); + } + + // Create a latch that we can receive a signal from when the query is running + this.queryRunningLatchRef.set(new CountDownLatch(1)); + + // Create a latch that we can send a signal to allow the query to finish + this.queryExitLatchRef.set(new CountDownLatch(1)); + + // Prepare a Callable that will compile and execute the XQuery (which will wait on the queryExitLatch) + final Callable latchedQueryCallable = () -> { + try (final DBBroker broker = brokerPool.get(Optional.of(brokerPool.getSecurityManager().getSystemSubject())); + final Txn transaction = brokerPool.getTransactionManager().beginTransaction(); + final Collection testCollection = broker.getOrCreateCollection(transaction, TEST_COLLECTION_URI)) { + + // Retrieve and READ LOCK our query's source document + final LockedDocument lockedQuerySourceDocument = testCollection.getDocumentWithLock(broker, LATCHED_QUERY_URI, Lock.LockMode.READ_LOCK); + this.lockedQuerySourceDocumentRef.set(lockedQuerySourceDocument); + + // NOTE: early release of Collection lock inline with Asymmetrical Locking scheme + testCollection.close(); + + // Compile our XQuery + final XQuery xquery = brokerPool.getXQueryService(); + final CompiledXQuery compiledQuery; + final XQueryContext xqueryContext = new XQueryContext(brokerPool); + + // Bind the latch references as external variables for the query + xqueryContext.declareVariable(new QName(RUNNING_LATCH_REF_XQUERY_VARIABLE_NAME), true, this.queryRunningLatchRef); + xqueryContext.declareVariable(new QName(EXIT_LATCH_REF_XQUERY_VARIABLE_NAME), true, this.queryExitLatchRef); + + final Source querySource = DbUriSource.from(broker.getBrokerPool(), broker.getCurrentSubject(), lockedQuerySourceDocumentRef.get().getDocument(), true, false); + compiledQuery = xquery.compile(xqueryContext, querySource); + + final Sequence result = xquery.execute(broker, compiledQuery, null); + + transaction.commit(); + + return result; + } catch (final XPathException e) { + e.printStackTrace(); + throw e; + } + }; + + // Run the latched query callable from a separate thread so that we don't block our test + this.executorService = Executors.newFixedThreadPool(2); // We size to 2 threads so that we have one for the latched query, and one for the test method itself (see tests below) + this.queryResultFuture = executorService.submit(latchedQueryCallable); + + // Wait for the query to start running + final CountDownLatch latch = this.queryRunningLatchRef.get(); + try { + final boolean running = latch.await(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertTrue("Timeout exceeded whilst waiting for query to notify us it is running", running); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); // Restore interrupted flag + throw e; + } + } + + @After + public void stopXQuery() { + @Nullable final CountDownLatch runningLatch = this.queryRunningLatchRef.getAndSet(null); + if (runningLatch != null) { + while (runningLatch.getCount() > 0) { + runningLatch.countDown(); + } + } + + @Nullable final CountDownLatch exitLatch = this.queryExitLatchRef.getAndSet(null); + if (exitLatch != null) { + while (exitLatch.getCount() > 0) { + exitLatch.countDown(); + } + } + + @Nullable final LockedDocument doc = this.lockedQuerySourceDocumentRef.getAndSet(null); + if (doc != null) { + final XmldbURI docUri = doc.getDocument().getURI(); + final LockManager lockManager = existEmbeddedServer.getBrokerPool().getLockManager(); + if (lockManager.isDocumentLockedForRead(docUri) || lockManager.isDocumentLockedForWrite(docUri)) { + doc.close(); + } + } + + this.queryResultFuture.cancel(true); + this.queryResultFuture = null; + this.executorService.shutdownNow(); + this.executorService = null; + } + + /** + * When this test is run, the /db/running-xquery-test/latched-query.xq XQuery + * is already executing (see {@link #runXQuery()}) in a separate thread that holds a READ_LOCK on its Source document. + * + * We try and read the XQuery source document under those conditions, and the read should succeed. + */ + @Test + public void readRunningXQuerySourceDocument() throws InterruptedException, ExecutionException, TimeoutException { + // Prepare a Callable that will try and read the XQuery's source document + final Callable readDocumentCallable = () -> { + + final BrokerPool brokerPool = existEmbeddedServer.getBrokerPool(); + + // try and read the XQuery source document + try (final DBBroker broker = brokerPool.get(Optional.of(brokerPool.getSecurityManager().getSystemSubject())); + final Txn transaction = brokerPool.getTransactionManager().beginTransaction(); + final Collection testCollection = broker.getOrCreateCollection(transaction, TEST_COLLECTION_URI)) { + + final String documentContent; + try (final LockedDocument lockedDocument = testCollection.getDocumentWithLock(broker, LATCHED_QUERY_URI, Lock.LockMode.READ_LOCK)) { + final DocumentImpl document = lockedDocument.getDocument(); + assertTrue(document instanceof BinaryDocument); + + // NOTE: early release of Collection lock inline with Asymmetrical Locking scheme + testCollection.close(); + + final BinaryDocument binaryDocument = ((BinaryDocument) document); + final BlobId blobId = binaryDocument.getBlobId(); + try (final InputStream is = brokerPool.getBlobStore().get(transaction, blobId)) { + documentContent = new String(readAll(is), StandardCharsets.UTF_8); + } + } + + transaction.commit(); + + return documentContent; + + } finally { + + // Instruct the running XQuery to finish + @Nullable final CountDownLatch exitLatch = queryExitLatchRef.get(); + assertNotNull("exitLatch should not be null at this point", exitLatch); + exitLatch.countDown(); + } + }; + + // Run the readDocumentCallable from a separate thread so that we don't block our test + final Future documentContentFuture = executorService.submit(readDocumentCallable); + final String documentContent = documentContentFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertEquals(query, documentContent); + + // Check the results of the query + final Sequence queryResult = queryResultFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertNotNull(queryResult); + assertEquals(1, queryResult.getItemCount()); + final Item resultItem = queryResult.itemAt(0); + assertEquals(Type.ELEMENT, resultItem.getType()); + final Element resultElement = (Element) resultItem; + assertTrue(Boolean.parseBoolean(resultElement.getAttribute("exit-latch-zero"))); + assertTrue(resultElement.getTextContent().startsWith("PT")); + } + + /** + * When this test is run, the /db/running-xquery-test/latched-query.xq XQuery + * is already executing (see {@link #runXQuery()}) in a separate thread that holds a READ_LOCK on its Source document. + * + * We try and replace the XQuery source document under those conditions, which should not be possible + * as writing to the document would require a WRITE_LOCK, but a READ_LOCK is already held. + */ + @Test + public void replaceRunningXQuerySourceDocument() { + // Prepare a Callable that will try and replace the XQuery's source document + final Callable replaceDocumentCallable = () -> { + + final BrokerPool brokerPool = existEmbeddedServer.getBrokerPool(); + + // try and replace the XQuery source document + try (final DBBroker broker = brokerPool.get(Optional.of(brokerPool.getSecurityManager().getSystemSubject())); + final Txn transaction = brokerPool.getTransactionManager().beginTransaction(); + final Collection testCollection = broker.getOrCreateCollection(transaction, TEST_COLLECTION_URI)) { + + final String replacementQuery = ""; + + final MediaType xqueryMediaType = broker.getBrokerPool().getMediaTypeService().getMediaTypeResolver().fromString(MediaType.APPLICATION_XQUERY); + broker.storeDocument(transaction, LATCHED_QUERY_URI, new StringInputSource(replacementQuery.getBytes(StandardCharsets.UTF_8)), xqueryMediaType, testCollection); + + // NOTE(AR) It should not have been possible to get to this point as we should not be able to acquire a WRITE_LOCK lock on the document (above) when calling broker.storeDocument, as the query thread holds a READ_LOCK on the document + + transaction.commit(); + + return true; + } + }; + + // Run the replaceDocumentCallable from a separate thread so that we don't block our test + final Future replacedDocumentFuture = executorService.submit(replaceDocumentCallable); + assertThrows( + "We should not have been able to replace the document as that requires this thread to obtain a WRITE_LOCK on the document, but the query thread already holds a READ_LOCK on the document", + TimeoutException.class, () -> + replacedDocumentFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS) + ); + } +} diff --git a/exist-core/src/test/java/org/exist/xquery/RunningXQueryXmlDbTest.java b/exist-core/src/test/java/org/exist/xquery/RunningXQueryXmlDbTest.java new file mode 100644 index 0000000000..8ecbab50cb --- /dev/null +++ b/exist-core/src/test/java/org/exist/xquery/RunningXQueryXmlDbTest.java @@ -0,0 +1,309 @@ +/* + * Copyright (C) 2014, Evolved Binary Ltd + * + * This file was originally ported from FusionDB to Elemental by + * Evolved Binary, for the benefit of the Elemental Open Source community. + * Only the ported code as it appears in this file, at the time that + * it was contributed to Elemental, was re-licensed under The GNU + * Lesser General Public License v2.1 only for use in Elemental. + * + * This license grant applies only to a snapshot of the code as it + * appeared when ported, it does not offer or infer any rights to either + * updates of this source code or access to the original source code. + * + * The GNU Lesser General Public License v2.1 only license follows. + * + * ===================================================================== + * + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.TestUtils; +import org.exist.storage.BrokerPool; +import org.exist.test.ExistWebServer; +import org.exist.xmldb.EXistResource; +import org.exist.xmldb.EXistResourceSet; +import org.exist.xmldb.EXistXPathQueryService; +import org.exist.xmldb.XmldbURI; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.xmldb.api.DatabaseManager; +import org.xmldb.api.base.Collection; +import org.xmldb.api.base.Resource; +import org.xmldb.api.base.XMLDBException; +import org.xmldb.api.modules.BinaryResource; +import org.xmldb.api.modules.CollectionManagementService; +import xyz.elemental.mediatype.MediaType; + +import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.exist.util.PropertiesBuilder.propertiesBuilder; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Performs a number of tests against the read-locked source document of an XQuery that is currently running over the XML-RPC API. + * + * @author Adam Retter + */ +@RunWith(Parameterized.class) +public class RunningXQueryXmlDbTest { + + private static final XmldbURI TEST_COLLECTION_URI = XmldbURI.DB.append("running-xquery-test"); + private static final XmldbURI LATCHED_QUERY_URI = XmldbURI.create("latched-query.xq"); + private static final String RUNNING_LATCH_REF_XQUERY_VARIABLE_NAME = "query-running-latch-ref"; + private static final String EXIT_LATCH_REF_XQUERY_VARIABLE_NAME = "query-exit-latch-ref"; + + private static final long TIMEOUT_MS = 3000; + private static final String PORT_PLACEHOLDER = "${PORT}"; + + @Parameterized.Parameters(name = "{0}") + public static java.util.Collection data() { + return Arrays.asList(new Object[][] { + { "local", "xmldb:exist://" }, + { "remote", "xmldb:exist://localhost:" + PORT_PLACEHOLDER + "/xmlrpc" } + }); + } + + @Parameterized.Parameter + public String apiName; + + @Parameterized.Parameter(value = 1) + public String baseUri; + + @Rule + public final ExistWebServer existWebServer = new ExistWebServer(true, false, + propertiesBuilder() + .put(FunctionFactory.PROPERTY_ENABLE_JAVA_BINDING, true) + .put(BrokerPool.PROPERTY_SHUTDOWN_DELAY, 100L) + .build(), + true, true, true); + + @Nullable private ExecutorService executorService = null; + @Nullable private Future queryResultFuture = null; + + private final String query = + "declare namespace system = 'http://exist-db.org/xquery/system';\n" + + "declare namespace lws = 'java:" + LatchWrappersSingleton.class.getName() + "';\n" + + "declare namespace at = 'java:" + AtomicReference.class.getName() + "';\n" + + "declare namespace cl = 'java:" + CountDownLatch.class.getName() + "';\n" + + "declare namespace tu = 'java:" + TimeUnit.class.getName() + "';\n" + + "declare variable $lwsi := lws:INSTANCE();\n" + + "declare variable $" + RUNNING_LATCH_REF_XQUERY_VARIABLE_NAME + " := lws:queryRunningLatchRef($lwsi);\n" + + "declare variable $" + EXIT_LATCH_REF_XQUERY_VARIABLE_NAME + " := lws:queryExitLatchRef($lwsi);\n" + + "\n" + + "let $running-latch := at:get($" + RUNNING_LATCH_REF_XQUERY_VARIABLE_NAME + ")\n" + + "let $_ := cl:countDown($running-latch)\n" + + "let $start := util:system-time()\n" + + "let $exit-latch := at:get($" + EXIT_LATCH_REF_XQUERY_VARIABLE_NAME + ")\n" + + "let $exit-latch-zero := cl:await($exit-latch, " + TIMEOUT_MS + " cast as xs:long, tu:MILLISECONDS())\n" + + "let $end := util:system-time()\n" + + "return\n" + + " {$end - $start}"; + + /** + * This singleton allows us to share access to the latches between this test in Java and the XQuery we are executing. + */ + public static class LatchWrappersSingleton { + public static final LatchWrappersSingleton INSTANCE = new LatchWrappersSingleton(); + + final AtomicReference queryRunningLatchRef = new AtomicReference<>(); + final AtomicReference queryExitLatchRef = new AtomicReference<>(); + + private LatchWrappersSingleton() { + } + } + + private String getBaseUri() { + return baseUri.replace(PORT_PLACEHOLDER, Integer.toString(existWebServer.getPort())); + } + + @Before + public void runXQuery() throws InterruptedException, XMLDBException { + // Store our XQuery as document into the database + try (final Collection root = DatabaseManager.getCollection(getBaseUri() + TEST_COLLECTION_URI.removeLastSegment().getCollectionPath(), TestUtils.ADMIN_DB_USER, TestUtils.ADMIN_DB_PWD)) { + final CollectionManagementService cms = (CollectionManagementService) root.getService("CollectionManagementService", "1.0"); + try (final Collection testCollection = cms.createCollection(TEST_COLLECTION_URI.lastSegmentString()); + final EXistResource queryResource = (EXistResource) testCollection.createResource(LATCHED_QUERY_URI.getCollectionPath(), BinaryResource.RESOURCE_TYPE)) { + queryResource.setMediaType(MediaType.APPLICATION_XQUERY); + queryResource.setContent(query.getBytes(UTF_8)); + testCollection.storeResource(queryResource); + } + } + + // Create a latch that we can receive a signal from when the query is running + LatchWrappersSingleton.INSTANCE.queryRunningLatchRef.set(new CountDownLatch(1)); + + // Create a latch that we can send a signal to allow the query to finish + LatchWrappersSingleton.INSTANCE.queryExitLatchRef.set(new CountDownLatch(1)); + + // Prepare a Callable that will compile and execute the XQuery (which will wait on the queryExitLatch) + final Callable latchedQueryCallable = () -> { + try (final Collection testCollection = DatabaseManager.getCollection(getBaseUri() + TEST_COLLECTION_URI.getCollectionPath(), TestUtils.ADMIN_DB_USER, TestUtils.ADMIN_DB_PWD)) { + final EXistXPathQueryService xpathQueryService = (EXistXPathQueryService) testCollection.getService("XPathQueryService", "1.0"); + try (final EXistResourceSet result = xpathQueryService.executeStoredQuery(TEST_COLLECTION_URI.append(LATCHED_QUERY_URI).getCollectionPath())) { + final Resource resultResource = result.getResource(0); + return (String) resultResource.getContent(); + } + } catch (final XMLDBException e) { + e.printStackTrace(); + throw e; + } + }; + + // Run the latched query callable from a separate thread so that we don't block our test + this.executorService = Executors.newFixedThreadPool(2); // We size to 2 threads so that we have one for the latched query, and one for the test method itself (see tests below) + this.queryResultFuture = executorService.submit(latchedQueryCallable); + + // Wait for the query to start running + final CountDownLatch latch = LatchWrappersSingleton.INSTANCE.queryRunningLatchRef.get(); + try { + final boolean running = latch.await(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertTrue("Timeout exceeded whilst waiting for query to notify us it is running", running); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); // Restore interrupted flag + throw e; + } + } + + @After + public void stopXQuery() { + @Nullable final CountDownLatch runningLatch = LatchWrappersSingleton.INSTANCE.queryRunningLatchRef.getAndSet(null); + if (runningLatch != null) { + while (runningLatch.getCount() > 0) { + runningLatch.countDown(); + } + } + + @Nullable final CountDownLatch exitLatch = LatchWrappersSingleton.INSTANCE.queryExitLatchRef.getAndSet(null); + if (exitLatch != null) { + while (exitLatch.getCount() > 0) { + exitLatch.countDown(); + } + } + + this.queryResultFuture.cancel(true); + this.queryResultFuture = null; + this.executorService.shutdownNow(); + this.executorService = null; + } + + /** + * When this test is run, the /db/running-xquery-test/latched-query.xq XQuery + * is already executing (see {@link #runXQuery()}) in a separate thread that holds a READ_LOCK on its Source document. + * + * We try and read the XQuery source document under those conditions, and the read should succeed. + */ + @Test + public void readRunningXQuerySourceDocument() throws InterruptedException, ExecutionException, TimeoutException { + // Prepare a Callable that will try and read the XQuery's source document + final Callable readDocumentCallable = () -> { + + // try and read the XQuery source document + try { + try (final Collection testCollection = DatabaseManager.getCollection(getBaseUri() + TEST_COLLECTION_URI.getCollectionPath(), TestUtils.ADMIN_DB_USER, TestUtils.ADMIN_DB_PWD); + final EXistResource queryResource = (EXistResource) testCollection.getResource(LATCHED_QUERY_URI.getCollectionPath())) { + assertTrue(queryResource instanceof BinaryResource); + return new String((byte[]) queryResource.getContent(), UTF_8); + } + } finally { + + // Instruct the running XQuery to finish + @Nullable final CountDownLatch exitLatch = LatchWrappersSingleton.INSTANCE.queryExitLatchRef.get(); + assertNotNull("exitLatch should not be null at this point", exitLatch); + exitLatch.countDown(); + } + }; + + // Run the readDocumentCallable from a separate thread so that we don't block our test + final Future documentContentFuture = executorService.submit(readDocumentCallable); + final String documentContent = documentContentFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertEquals(query, documentContent); + + // Check the results of the query + final String queryResult = queryResultFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertNotNull(queryResult); + assertTrue(queryResult.startsWith("PT")); + assertTrue(queryResult.endsWith("")); + } + + /** + * When this test is run, the /db/running-xquery-test/latched-query.xq XQuery + * is already executing (see {@link #runXQuery()}) in a separate thread, + * that should have released the READ_LOCK on its Source document after query compilation. + * + * We try and replace the XQuery source document under those conditions, which should be possible + * as no lock is held on the source document after compilation. + */ + @Test + public void replaceRunningXQuerySourceDocument() throws ExecutionException, InterruptedException, TimeoutException { + // Prepare a Callable that will try and replace the XQuery's source document + final Callable replaceDocumentCallable = () -> { + + try { + final String replacementQuery = ""; + + try (final Collection testCollection = DatabaseManager.getCollection(getBaseUri() + TEST_COLLECTION_URI.getCollectionPath(), TestUtils.ADMIN_DB_USER, TestUtils.ADMIN_DB_PWD); + final EXistResource queryResource = (EXistResource) testCollection.getResource(LATCHED_QUERY_URI.getCollectionPath())) { + queryResource.setMediaType(MediaType.APPLICATION_XQUERY); + queryResource.setContent(replacementQuery.getBytes(UTF_8)); + testCollection.storeResource(queryResource); + } + + return true; + + } finally { + // Instruct the running XQuery to finish + @Nullable final CountDownLatch exitLatch = LatchWrappersSingleton.INSTANCE.queryExitLatchRef.get(); + assertNotNull("exitLatch should not be null at this point", exitLatch); + exitLatch.countDown(); + } + }; + + // Run the replaceDocumentCallable from a separate thread so that we don't block our test + final Future replacedDocumentFuture = executorService.submit(replaceDocumentCallable); + final Boolean replaceDocumentStatus = replacedDocumentFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertTrue(replaceDocumentStatus); + + // Check the results of the query + // Check the results of the query + final String queryResult = queryResultFuture.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertNotNull(queryResult); + assertTrue(queryResult.startsWith("PT")); + assertTrue(queryResult.endsWith("")); + } +} diff --git a/exist-core/src/test/java/org/exist/xquery/XQueryContextAttributesTest.java b/exist-core/src/test/java/org/exist/xquery/XQueryContextAttributesTest.java index bb1ceb1aeb..54c7582473 100644 --- a/exist-core/src/test/java/org/exist/xquery/XQueryContextAttributesTest.java +++ b/exist-core/src/test/java/org/exist/xquery/XQueryContextAttributesTest.java @@ -32,17 +32,19 @@ */ package org.exist.xquery; -import com.evolvedbinary.j8fu.tuple.Tuple2; +import com.evolvedbinary.j8fu.function.BiConsumerE; +import com.evolvedbinary.j8fu.function.ConsumerE; import org.exist.EXistException; import org.exist.collections.Collection; import org.exist.dom.persistent.BinaryDocument; import org.exist.security.PermissionDeniedException; -import org.exist.source.DBSource; +import org.exist.source.DbUriSource; import org.exist.storage.BrokerPool; import org.exist.storage.DBBroker; import org.exist.storage.lock.Lock; import org.exist.storage.txn.Txn; import org.exist.test.ExistEmbeddedServer; +import org.exist.util.Holder; import org.exist.util.LockException; import org.exist.util.StringInputSource; import org.exist.xmldb.XmldbURI; @@ -57,9 +59,6 @@ import java.util.Optional; import static java.nio.charset.StandardCharsets.UTF_8; -import static com.evolvedbinary.j8fu.tuple.Tuple.Tuple; -import static org.exist.xquery.XQueryUtil.executeQuery; -import static org.exist.xquery.XQueryUtil.withCompiledQuery; import static org.junit.Assert.*; /** @@ -78,25 +77,28 @@ public void attributesOfMainModuleContextCleared() throws EXistException, LockEx final XmldbURI mainQueryUri = XmldbURI.create("/db/query1.xq"); final InputSource mainQuery = new StringInputSource("".getBytes(UTF_8)); - final DBSource mainQuerySource = storeQuery(broker, transaction, mainQueryUri, mainQuery); - - final XQueryContext escapedMainQueryContext = withCompiledQuery(broker, mainQuerySource, mainCompiledQuery -> { - final XQueryContext mainQueryContext = mainCompiledQuery.getContext(); - - mainQueryContext.setAttribute("attr1", "value1"); - mainQueryContext.setAttribute("attr2", "value2"); - - // execute the query - final Sequence result = executeQuery(broker, mainCompiledQuery); + final DbUriSource mainQuerySource = storeQuery(broker, transaction, mainQueryUri, mainQuery); + + // set some attributes on the context before query execution + final ConsumerE preExecutionContext = xQueryContext -> { + xQueryContext.setAttribute("attr1", "value1"); + xQueryContext.setAttribute("attr2", "value2"); + }; + + // will hold whether the context attributes is empty once the query has finished executing + final Holder attributesIsEmptyHolder = new Holder<>(); + final BiConsumerE postExecutionContext = (xqueryContext, result) -> { + attributesIsEmptyHolder.value = xqueryContext.attributes.isEmpty(); + }; + + // execute the query + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, mainQuerySource, false, null, null, null, preExecutionContext, postExecutionContext)) { + final Sequence result = queryResult.result; assertEquals(1, result.getItemCount()); + } - // intentionally escape the context from the lambda - return mainQueryContext; - }); - - assertNull(escapedMainQueryContext.getAttribute("attr1")); - assertNull(escapedMainQueryContext.getAttribute("attr2")); - assertTrue(escapedMainQueryContext.attributes.isEmpty()); + // now the query has finished executing and been reset, check the attributes map is empty for the Main Module + assertTrue(attributesIsEmptyHolder.value); transaction.commit(); } @@ -120,53 +122,53 @@ public void attributesOfLibraryModuleContextCleared() throws EXistException, Loc ("import module namespace mod1 = 'http://mod1' at 'xmldb:exist://" + libraryQueryUri + "';\n" + "mod1:f1()").getBytes(UTF_8) ); - final DBSource mainQuerySource = storeQuery(broker, transaction, mainQueryUri, mainQuery); + final DbUriSource mainQuerySource = storeQuery(broker, transaction, mainQueryUri, mainQuery); - final Tuple2 escapedContexts = withCompiledQuery(broker, mainQuerySource, mainCompiledQuery -> { - final XQueryContext mainQueryContext = mainCompiledQuery.getContext(); - // get the context of the library module - final Module[] libraryModules = mainQueryContext.getModules("http://mod1"); - assertEquals(1, libraryModules.length); - assertTrue(libraryModules[0] instanceof ExternalModule); + // set some attributes on the context in the Library Module before query execution + final ConsumerE preExecutionContext = mainModuleXqueryContext -> { + final Module[] libraryModules = mainModuleXqueryContext.getModules("http://mod1"); final ExternalModule libraryModule = (ExternalModule) libraryModules[0]; - final XQueryContext libraryQueryContext = libraryModule.getContext(); - assertTrue(libraryQueryContext instanceof ModuleContext); - - libraryQueryContext.setAttribute("attr1", "value1"); - libraryQueryContext.setAttribute("attr2", "value2"); + final XQueryContext libraryModuleXqueryContext = libraryModule.getContext(); + libraryModuleXqueryContext.setAttribute("attr1", "value1"); + libraryModuleXqueryContext.setAttribute("attr2", "value2"); + }; + + // will hold whether the context attributes in the Main Module is empty once the query has finished executing + final Holder mainModuleAttributesIsEmptyHolder = new Holder<>(); + // will hold whether the context attributes in the Library Module is empty once the query has finished executing + final Holder libraryModuleAttributesIsEmptyHolder = new Holder<>(); + final BiConsumerE postExecutionContext = (mainModuleXqueryContext, result) -> { + mainModuleAttributesIsEmptyHolder.value = mainModuleXqueryContext.attributes.isEmpty(); + + final Module[] libraryModules = mainModuleXqueryContext.getModules("http://mod1"); + final ExternalModule libraryModule = (ExternalModule) libraryModules[0]; + final XQueryContext libraryModuleXqueryContext = libraryModule.getContext(); + libraryModuleAttributesIsEmptyHolder.value = libraryModuleXqueryContext.attributes.isEmpty(); + }; - // execute the query - final Sequence result = executeQuery(broker, mainCompiledQuery); + // execute the query + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, mainQuerySource, false, null, null, null, preExecutionContext, postExecutionContext)) { + final Sequence result = queryResult.result; assertEquals(1, result.getItemCount()); + } - // intentionally escape the contexts from the lambda - return Tuple(mainQueryContext, (ModuleContext) libraryQueryContext); - }); - - final XQueryContext escapedMainQueryContext = escapedContexts._1; - final ModuleContext escapedLibraryQueryContext = escapedContexts._2; - assertTrue(escapedMainQueryContext != escapedLibraryQueryContext); + // now the query has finished executing and been reset, check the attributes map is empty for the Main Module + assertTrue(mainModuleAttributesIsEmptyHolder.value); - assertNull(escapedMainQueryContext.getAttribute("attr1")); - assertNull(escapedMainQueryContext.getAttribute("attr2")); - assertTrue(escapedMainQueryContext.attributes.isEmpty()); - - assertNull(escapedLibraryQueryContext.getAttribute("attr1")); - assertNull(escapedLibraryQueryContext.getAttribute("attr2")); - assertTrue(escapedLibraryQueryContext.attributes.isEmpty()); + // now the query has finished executing and been reset, check the attributes map is empty for the Library Module + assertTrue(libraryModuleAttributesIsEmptyHolder.value); transaction.commit(); } } - private static DBSource storeQuery(final DBBroker broker, final Txn transaction, final XmldbURI uri, final InputSource source) throws IOException, PermissionDeniedException, SAXException, LockException, EXistException { + private static DbUriSource storeQuery(final DBBroker broker, final Txn transaction, final XmldbURI uri, final InputSource source) throws IOException, PermissionDeniedException, SAXException, LockException, EXistException { try (final Collection collection = broker.openCollection(uri.removeLastSegment(), Lock.LockMode.WRITE_LOCK)) { final MediaType xqueryMediaType = broker.getBrokerPool().getMediaTypeService().getMediaTypeResolver().fromString(MediaType.APPLICATION_XQUERY); broker.storeDocument(transaction, uri.lastSegment(), source, xqueryMediaType, collection); final BinaryDocument doc = (BinaryDocument) collection.getDocument(broker, uri.lastSegment()); - - return new DBSource(broker, doc, false); + return DbUriSource.from(broker.getBrokerPool(), broker.getCurrentSubject(), doc, false, false); } } } diff --git a/exist-core/src/test/java/org/exist/xquery/XmldbBinariesTest.java b/exist-core/src/test/java/org/exist/xquery/XmldbBinariesTest.java index 0d4874f6e9..2947dfd9b7 100644 --- a/exist-core/src/test/java/org/exist/xquery/XmldbBinariesTest.java +++ b/exist-core/src/test/java/org/exist/xquery/XmldbBinariesTest.java @@ -158,17 +158,9 @@ protected QueryResultAccessor executeXQuery(fi final XQueryService xqueryService = (XQueryService)colRoot.getService("XQueryService", "1.0"); final CompiledExpression compiledExpression = xqueryService.compile(query); - final EXistResourceSet results = (EXistResourceSet) xqueryService.execute(compiledExpression); - - - try { + try (final EXistResourceSet results = (EXistResourceSet) xqueryService.execute(compiledExpression)) { // compiledExpression.reset(); // shows the ordering issue with binary values (see comment below) - consumer.accept(results); - } finally { - //the following calls cause the streams of any binary result values to be closed, so if we did so before we are finished with the results, serialization would fail. - results.clear(); - compiledExpression.reset(); } } }; diff --git a/exist-jetty-config/src/main/resources/standalone-webapp/WEB-INF/web.xml b/exist-jetty-config/src/main/resources/standalone-webapp/WEB-INF/web.xml index 9b3140a627..acbac33b53 100644 --- a/exist-jetty-config/src/main/resources/standalone-webapp/WEB-INF/web.xml +++ b/exist-jetty-config/src/main/resources/standalone-webapp/WEB-INF/web.xml @@ -4,7 +4,6 @@ | to the XML-RPC, REST and WebDAV servlets, plus XQueryURLRewrite for the URL | handling. Use this configuration if you need a minimum setup with web content | stored in the db. - | $Id$ +--> - diff --git a/exist-samples/src/main/resources/org/exist/samples/http/client.sh b/exist-samples/src/main/resources/org/exist/samples/http/client.sh index 7468adeb26..8f0f99a2a2 100644 --- a/exist-samples/src/main/resources/org/exist/samples/http/client.sh +++ b/exist-samples/src/main/resources/org/exist/samples/http/client.sh @@ -28,7 +28,6 @@ # # Author: Wolfgang Meier # -# $Id$ url=http://localhost:8080/exist/rest diff --git a/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook.xsd b/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook.xsd index 875a9b8a12..9c03a7b48e 100644 --- a/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook.xsd +++ b/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook.xsd @@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA --> - diff --git a/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook_invalid.xml b/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook_invalid.xml index d211d83d75..65181b7176 100644 --- a/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook_invalid.xml +++ b/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook_invalid.xml @@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA --> - John Punin diff --git a/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook_valid.xml b/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook_valid.xml index 6d570df3fd..23f22f4cf1 100644 --- a/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook_valid.xml +++ b/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/addressbook_valid.xml @@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA --> - John Punin diff --git a/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/catalog.xml b/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/catalog.xml index 1f1ce5d90a..04e314cdb7 100644 --- a/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/catalog.xml +++ b/exist-samples/src/main/resources/org/exist/samples/validation/addressbook/catalog.xml @@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA --> - \ No newline at end of file diff --git a/exist-samples/src/main/resources/org/exist/samples/validation/dtd/catalog.xml b/exist-samples/src/main/resources/org/exist/samples/validation/dtd/catalog.xml index c256326bdf..dcb8b7759d 100644 --- a/exist-samples/src/main/resources/org/exist/samples/validation/dtd/catalog.xml +++ b/exist-samples/src/main/resources/org/exist/samples/validation/dtd/catalog.xml @@ -1,5 +1,4 @@ - \ No newline at end of file diff --git a/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_invalid.xml b/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_invalid.xml index 66520059b2..cf809b7746 100644 --- a/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_invalid.xml +++ b/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_invalid.xml @@ -1,6 +1,5 @@ - The Tragedy of Hamlet, Prince of Denmark diff --git a/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_nodoctype.xml b/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_nodoctype.xml index 5ac0ca554c..646d06612e 100644 --- a/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_nodoctype.xml +++ b/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_nodoctype.xml @@ -1,5 +1,4 @@ - The Tragedy of Hamlet, Prince of Denmark diff --git a/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_valid.xml b/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_valid.xml index ee8c583d53..66cbf0029b 100644 --- a/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_valid.xml +++ b/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_valid.xml @@ -1,6 +1,5 @@ - The Tragedy of Hamlet, Prince of Denmark diff --git a/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_wrongdoctype.xml b/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_wrongdoctype.xml index d3e5f9097e..59780d6e4a 100644 --- a/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_wrongdoctype.xml +++ b/exist-samples/src/main/resources/org/exist/samples/validation/dtd/hamlet_wrongdoctype.xml @@ -1,6 +1,5 @@ - The Tragedy of Hamlet, Prince of Denmark diff --git a/exist-samples/src/main/resources/org/exist/samples/validation/parse/catalog.xml b/exist-samples/src/main/resources/org/exist/samples/validation/parse/catalog.xml index 4b8b9d1342..33ae49b7d6 100644 --- a/exist-samples/src/main/resources/org/exist/samples/validation/parse/catalog.xml +++ b/exist-samples/src/main/resources/org/exist/samples/validation/parse/catalog.xml @@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA --> - diff --git a/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/ResourceFunctionExecutorImpl.java b/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/ResourceFunctionExecutorImpl.java index 034d6df547..f23cacabc3 100644 --- a/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/ResourceFunctionExecutorImpl.java +++ b/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/ResourceFunctionExecutorImpl.java @@ -27,6 +27,7 @@ package org.exist.extensions.exquery.restxq.impl; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -44,7 +45,6 @@ import org.exist.security.EffectiveSubject; import org.exist.security.Permission; import org.exist.security.PermissionDeniedException; -import org.exist.source.DBSource; import org.exist.source.Source; import org.exist.storage.BrokerPool; import org.exist.storage.DBBroker; @@ -217,28 +217,27 @@ public Sequence execute(final ResourceFunction resourceFunction, final Iterable< * @param xquery The XQuery to determine the effective subject for * @return Maybe an effective subject or empty if there is no setUid or setGid bits */ - private Optional getEffectiveSubject(final CompiledXQuery xquery) { + private Optional getEffectiveSubject(final CompiledXQuery xquery) throws PermissionDeniedException { final Optional effectiveSubject; final Source src = xquery.getContext().getSource(); - if(src instanceof DBSource) { - final DBSource dbSrc = (DBSource)src; - final Permission perm = dbSrc.getPermissions(); - - if(perm.isSetUid()) { - if(perm.isSetGid()) { - //setUid and SetGid - effectiveSubject = Optional.of(new EffectiveSubject(perm.getOwner(), perm.getGroup())); - } else { - //just setUid - effectiveSubject = Optional.of(new EffectiveSubject(perm.getOwner())); - } - } else if(perm.isSetGid()) { - //just setGid, so we use the current user as the effective user - effectiveSubject = Optional.of(new EffectiveSubject(xquery.getContext().getBroker().getCurrentSubject(), perm.getGroup())); + final Permission perm; + try { + perm = src.getPermissions(); + } catch (final IOException e) { + throw new PermissionDeniedException(e); + } + if(perm.isSetUid()) { + if(perm.isSetGid()) { + //setUid and SetGid + effectiveSubject = Optional.of(new EffectiveSubject(perm.getOwner(), perm.getGroup())); } else { - effectiveSubject = Optional.empty(); + //just setUid + effectiveSubject = Optional.of(new EffectiveSubject(perm.getOwner())); } + } else if(perm.isSetGid()) { + //just setGid, so we use the current user as the effective user + effectiveSubject = Optional.of(new EffectiveSubject(xquery.getContext().getBroker().getCurrentSubject(), perm.getGroup())); } else { effectiveSubject = Optional.empty(); } @@ -479,4 +478,4 @@ public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException public void dump(final ExpressionDumper dumper) { } } -} \ No newline at end of file +} diff --git a/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/XQueryCompiler.java b/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/XQueryCompiler.java index bdedb41f09..016ca47dbd 100644 --- a/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/XQueryCompiler.java +++ b/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/XQueryCompiler.java @@ -32,7 +32,8 @@ import org.exist.dom.persistent.DocumentImpl; import org.exist.security.Permission; import org.exist.security.PermissionDeniedException; -import org.exist.source.DBSource; +import org.exist.source.DbUriSource; +import org.exist.source.Source; import org.exist.storage.DBBroker; import org.exist.xmldb.XmldbURI; import org.exist.xquery.CompiledXQuery; @@ -70,10 +71,10 @@ public static CompiledXQuery compile(final DBBroker broker, final DocumentImpl d //compile the query final XQueryContext context = new XQueryContext(broker.getBrokerPool()); - final DBSource source = new DBSource(broker, (BinaryDocument)document, true); + final Source source = DbUriSource.from(broker.getBrokerPool(), broker.getCurrentSubject(), document, true, false); //set the module load path for any module imports that are relative - context.setModuleLoadPath(XmldbURI.EMBEDDED_SERVER_URI_PREFIX + source.getDocumentPath().removeLastSegment()); + context.setModuleLoadPath(XmldbURI.EMBEDDED_SERVER_URI_PREFIX + document.getURI().removeLastSegment()); return broker.getBrokerPool().getXQueryService().compile(context, source); } else { @@ -90,4 +91,4 @@ public static CompiledXQuery compile(final DBBroker broker, final DocumentImpl d throw new RestXqServiceCompilationException("Permission to access XQuery denied: " + document.getURI().toString() + ": " + pde.getMessage(), pde); } } -} \ No newline at end of file +} diff --git a/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/XQueryInspector.java b/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/XQueryInspector.java index 295ddfe7e4..e80b7e490b 100644 --- a/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/XQueryInspector.java +++ b/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/XQueryInspector.java @@ -36,7 +36,7 @@ import java.util.Map; import java.util.Set; import org.exist.extensions.exquery.restxq.impl.adapters.AnnotationAdapter; -import org.exist.source.DBSource; +import org.exist.source.DbStoreSource; import org.exist.source.Source; import org.exist.xquery.Annotation; import org.exist.xquery.CompiledXQuery; @@ -51,6 +51,8 @@ import org.exquery.restxq.impl.ResourceFunctionFactory; import org.exquery.restxq.impl.annotation.RestAnnotationFactory; +import javax.annotation.Nullable; + /** * * @author Adam Retter @@ -114,10 +116,9 @@ public static Map> getDependencies(final CompiledXQuery comp } private static void getDependencies(final XQueryContext xqyCtx, final Map> dependencies) { - - final String xqueryUri = getDbUri(xqyCtx.getSource()); - Set depSet = dependencies.get(xqueryUri); - if(depSet == null) { + @Nullable final String dbXqueryUri = getDbUri(xqyCtx.getSource()); + @Nullable Set depSet = dependencies.get(dbXqueryUri); + if (depSet == null) { final Iterator itModule = xqyCtx.getModules(); while(itModule.hasNext()) { @@ -125,19 +126,19 @@ private static void getDependencies(final XQueryContext xqyCtx, final Map(); } - depSet.add(moduleUri); + depSet.add(dbModuleUri); /* * must merge map here as recursive function * can cause problems with recursive * module imports m1 -> m2 -> m2 -> m1 */ - dependencies.put(xqueryUri, depSet); + dependencies.put(dbXqueryUri, depSet); } getDependencies(extModule.getContext(), dependencies); @@ -146,11 +147,10 @@ private static void getDependencies(final XQueryContext xqyCtx, final Mapsrc/main/java/org/exist/xquery/modules/compression/AbstractCompressFunction.java src/main/java/org/exist/xquery/modules/compression/AbstractExtractFunction.java src/main/java/org/exist/xquery/modules/compression/CompressionModule.java + src/test/java/xquery/modules/compression/CompressionTests.java src/main/java/org/exist/xquery/modules/compression/EntryFunctions.java src/main/java/org/exist/xquery/modules/compression/UnTarFunction.java src/main/java/org/exist/xquery/modules/compression/UnZipFunction.java @@ -192,6 +193,7 @@ src/main/java/org/exist/xquery/modules/compression/AbstractCompressFunction.java src/main/java/org/exist/xquery/modules/compression/AbstractExtractFunction.java src/main/java/org/exist/xquery/modules/compression/CompressionModule.java + src/test/java/xquery/modules/compression/CompressionTests.java src/main/java/org/exist/xquery/modules/compression/EntryFunctions.java src/main/java/org/exist/xquery/modules/compression/UnTarFunction.java src/main/java/org/exist/xquery/modules/compression/UnZipFunction.java diff --git a/extensions/modules/compression/src/test/java/xquery/modules/compression/CompressionTests.java b/extensions/modules/compression/src/test/java/xquery/modules/compression/CompressionTests.java index bda0fc719f..3a44f9e67d 100644 --- a/extensions/modules/compression/src/test/java/xquery/modules/compression/CompressionTests.java +++ b/extensions/modules/compression/src/test/java/xquery/modules/compression/CompressionTests.java @@ -1,4 +1,28 @@ /* + * Elemental + * Copyright (C) 2024, Evolved Binary Ltd + * + * admin@evolvedbinary.com + * https://www.evolvedbinary.com | https://www.elemental.xyz + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; version 2.1. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * NOTE: Parts of this file contain code from 'The eXist-db Authors'. + * The original license header is included below. + * + * ===================================================================== + * * eXist-db Open Source Native XML Database * Copyright (C) 2001 The eXist-db Authors * @@ -21,27 +45,6 @@ */ package xquery.modules.compression; -/* - * eXist Open Source Native XML Database - * Copyright (C) 2015 The eXist Project - * http://exist-db.org - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - * $Id$ - */ import org.exist.test.runner.XSuite; import org.junit.runner.RunWith; diff --git a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java index 4bdbb9e4b2..3a253f04f7 100644 --- a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java +++ b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java @@ -177,18 +177,10 @@ protected QueryResultAccessor executeXQuery(fi final XQueryService xqueryService = (XQueryService)colRoot.getService("XQueryService", "1.0"); final CompiledExpression compiledExpression = xqueryService.compile(query); - final EXistResourceSet results = (EXistResourceSet) xqueryService.execute(compiledExpression); - - - try { - // compiledExpression.reset(); // shows the ordering issue with binary values (see comment below) - - consumer.accept(results); - } finally { - //the following calls cause the streams of any binary result values to be closed, so if we did so before we are finished with the results, serialization would fail. - results.clear(); - compiledExpression.reset(); - } + try (final EXistResourceSet results = (EXistResourceSet) xqueryService.execute(compiledExpression)) { +// compiledExpression.reset(); // shows the ordering issue with binary values (see comment below) + consumer.accept(results); + } } finally { colRoot.close(); } diff --git a/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ConnectionIT.java b/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ConnectionIT.java index 1fdd351535..8f86b132e5 100644 --- a/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ConnectionIT.java +++ b/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ConnectionIT.java @@ -32,7 +32,7 @@ */ package org.exist.xquery.modules.sql; -import com.evolvedbinary.j8fu.tuple.Tuple2; +import com.evolvedbinary.j8fu.function.BiConsumerE; import org.exist.EXistException; import org.exist.collections.Collection; import org.exist.security.PermissionDeniedException; @@ -43,6 +43,7 @@ import org.exist.storage.lock.Lock; import org.exist.storage.txn.Txn; import org.exist.test.ExistEmbeddedServer; +import org.exist.util.Holder; import org.exist.util.LockException; import org.exist.util.StringInputSource; import org.exist.xmldb.XmldbURI; @@ -60,10 +61,7 @@ import java.util.Map; import java.util.Optional; -import static com.evolvedbinary.j8fu.tuple.Tuple.Tuple; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.exist.xquery.XQueryUtil.executeQuery; -import static org.exist.xquery.XQueryUtil.withCompiledQuery; import static org.junit.Assert.*; /** @@ -90,26 +88,28 @@ public void getConnectionIsAutomaticallyClosed() throws EXistException, XPathExc try (final DBBroker broker = pool.getBroker(); final Txn transaction = pool.getTransactionManager().beginTransaction()) { - final XQueryContext escapedMainQueryContext = withCompiledQuery(broker, mainQuerySource, mainCompiledQuery -> { - final XQueryContext mainQueryContext = mainCompiledQuery.getContext(); - - // execute the query - final Sequence result = executeQuery(broker, mainCompiledQuery); + // will hold the number of open connections once the query has finished executing + final Holder connectionsCountHolder = new Holder<>(); + final BiConsumerE postExecutionContext = (xqueryContext, result) -> { + final int connectionsCount = ModuleUtils.readContextMap(xqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + connectionsCountHolder.value = connectionsCount; + }; + // execute query + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, mainQuerySource, false, null, null, null, null, postExecutionContext)) { // check that the handle for the sql connection that was created was valid + final Sequence result = queryResult.result; assertEquals(1, result.getItemCount()); assertTrue(result.itemAt(0) instanceof IntegerValue); assertEquals(Type.LONG, result.itemAt(0).getType()); - final long connectionHandle = result.itemAt(0).toJavaObject(long.class); - assertFalse(connectionHandle == 0); - // intentionally escape the context from the lambda - return mainQueryContext; - }); + // check there is an active connection + final long connectionHandle = result.itemAt(0).toJavaObject(long.class); + assertNotEquals(0, connectionHandle); + } - // check the connections map is empty - final int connectionsCount = ModuleUtils.readContextMap(escapedMainQueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, connectionsCount); + // now the query has finished executing and been reset, check the connections map is empty + assertEquals(0, connectionsCountHolder.value.intValue()); transaction.commit(); } @@ -139,42 +139,40 @@ public void getConnectionFromModuleIsAutomaticallyClosed() throws EXistException broker.storeDocument(transaction, XmldbURI.create("mymodule.xqm"), new StringInputSource(moduleQuery.getBytes(UTF_8)), xqueryMediaType, collection); } - final Tuple2 escapedContexts = withCompiledQuery(broker, mainQuerySource, mainCompiledQuery -> { - final XQueryContext mainQueryContext = mainCompiledQuery.getContext(); + // will hold the number of open connections in the Main Module once the query has finished executing + final Holder mainModuleConnectionsCountHolder = new Holder<>(); + // will hold the number of open connections in the Library Module once the query has finished executing + final Holder libraryModuleConnectionsCountHolder = new Holder<>(); + final BiConsumerE postExecutionContext = (mainModuleXqueryContext, result) -> { + final int mainModuleConnectionsCount = ModuleUtils.readContextMap(mainModuleXqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + mainModuleConnectionsCountHolder.value = mainModuleConnectionsCount; - // get the context of the library module - final Module[] libraryModules = mainQueryContext.getModules("http://mymodule.com"); - assertEquals(1, libraryModules.length); - assertTrue(libraryModules[0] instanceof ExternalModule); + final Module[] libraryModules = mainModuleXqueryContext.getModules("http://mymodule.com"); final ExternalModule libraryModule = (ExternalModule) libraryModules[0]; - final XQueryContext libraryQueryContext = libraryModule.getContext(); - assertTrue(libraryQueryContext instanceof ModuleContext); + final XQueryContext libraryModuleXqueryContext = libraryModule.getContext(); + final int libraryModuleConnectionsCount = ModuleUtils.readContextMap(libraryModuleXqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + libraryModuleConnectionsCountHolder.value = libraryModuleConnectionsCount; + }; - // execute the query - final Sequence result = executeQuery(broker, mainCompiledQuery); + // execute query + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, mainQuerySource, false, null, null, null, null, postExecutionContext)) { // check that the handle for the sql connection that was created was valid + final Sequence result = queryResult.result; assertEquals(1, result.getItemCount()); assertTrue(result.itemAt(0) instanceof IntegerValue); assertEquals(Type.LONG, result.itemAt(0).getType()); - final long connectionHandle = result.itemAt(0).toJavaObject(long.class); - assertFalse(connectionHandle == 0); - // intentionally escape the contexts from the lambda - return Tuple(mainQueryContext, (ModuleContext) libraryQueryContext); - }); - - final XQueryContext escapedMainQueryContext = escapedContexts._1; - final ModuleContext escapedLibraryQueryContext = escapedContexts._2; - assertTrue(escapedMainQueryContext != escapedLibraryQueryContext); + // check there is an active connection + final long connectionHandle = result.itemAt(0).toJavaObject(long.class); + assertNotEquals(0, connectionHandle); + } - // check the connections were closed in the main module - final int mainConnectionsCount = ModuleUtils.readContextMap(escapedMainQueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, mainConnectionsCount); + // now the query has finished executing and been reset, check the connections map is empty for the Main Module + assertEquals(0, mainModuleConnectionsCountHolder.value.intValue()); - // check the connections were closed in the library module - final int libraryConnectionsCount = ModuleUtils.readContextMap(escapedLibraryQueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, libraryConnectionsCount); + // now the query has finished executing and been reset, check the connections map is empty for the Library Module + assertEquals(0, libraryModuleConnectionsCountHolder.value.intValue()); transaction.commit(); } @@ -192,18 +190,23 @@ public void getConnectionCanBeExplicitlyClosed() throws EXistException, XPathExc try (final DBBroker broker = pool.getBroker(); final Txn transaction = pool.getTransactionManager().beginTransaction()) { - // execute query - final Tuple2 contextAndResult = withCompiledQuery(broker, source, compiledXQuery -> { - final Sequence result = executeQuery(broker, compiledXQuery); - return Tuple(compiledXQuery.getContext(), result.itemAt(0).toJavaObject(boolean.class)); - }); + // will hold the number of open connections once the query has finished executing + final Holder connectionsCountHolder = new Holder<>(); + final BiConsumerE postExecutionContext = (xqueryContext, result) -> { + final int connectionsCount = ModuleUtils.readContextMap(xqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + connectionsCountHolder.value = connectionsCount; + }; - // check that the handle for the sql connection was closed - assertTrue(contextAndResult._2); + // execute query + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, source, false, null, null, null, null, postExecutionContext)) { + // check that the handle for the sql connection was closed + final Sequence result = queryResult.result; + final boolean connectionIsClosed = result.itemAt(0).toJavaObject(boolean.class); + assertTrue(connectionIsClosed); + } - // check the connections were closed - final int connectionsCount = ModuleUtils.readContextMap(contextAndResult._1, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, connectionsCount); + // now the query has finished executing and been reset, check the connections map is empty + assertEquals(0, connectionsCountHolder.value.intValue()); transaction.commit(); } diff --git a/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ConnectionPoolIT.java b/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ConnectionPoolIT.java index c47fc6c782..a8fba301c7 100644 --- a/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ConnectionPoolIT.java +++ b/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ConnectionPoolIT.java @@ -32,6 +32,7 @@ */ package org.exist.xquery.modules.sql; +import com.evolvedbinary.j8fu.function.BiConsumerE; import org.exist.EXistException; import org.exist.security.PermissionDeniedException; import org.exist.source.Source; @@ -40,8 +41,10 @@ import org.exist.storage.DBBroker; import org.exist.storage.txn.Txn; import org.exist.test.ExistEmbeddedServer; +import org.exist.util.Holder; import org.exist.xquery.XPathException; import org.exist.xquery.XQueryContext; +import org.exist.xquery.XQueryUtil; import org.exist.xquery.modules.ModuleUtils; import org.exist.xquery.value.IntegerValue; import org.exist.xquery.value.Sequence; @@ -52,8 +55,6 @@ import java.io.IOException; import java.util.Map; -import static org.exist.xquery.XQueryUtil.executeQuery; -import static org.exist.xquery.XQueryUtil.withCompiledQuery; import static org.junit.Assert.*; public class ConnectionPoolIT { @@ -73,27 +74,26 @@ public void getConnectionFromPoolIsAutomaticallyClosed() throws EXistException, try (final DBBroker broker = pool.getBroker(); final Txn transaction = pool.getTransactionManager().beginTransaction()) { - final XQueryContext escapedMainQueryContext = withCompiledQuery(broker, mainQuerySource, mainCompiledQuery -> { - final XQueryContext mainQueryContext = mainCompiledQuery.getContext(); - - // execute the query - final Sequence result = executeQuery(broker, mainCompiledQuery); - + // will hold the number of open connections once the query has finished executing + final Holder connectionsCountHolder = new Holder<>(); + final BiConsumerE postExecutionContext = (xqueryContext, result) -> { + final int connectionsCount = ModuleUtils.readContextMap(xqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + connectionsCountHolder.value = connectionsCount; + }; + // execute query + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, mainQuerySource, false, null, null, null, null, postExecutionContext)) { // check that the handle for the sql connection that was created was valid + final Sequence result = queryResult.result; assertEquals(1, result.getItemCount()); assertTrue(result.itemAt(0) instanceof IntegerValue); assertEquals(Type.LONG, result.itemAt(0).getType()); final long connectionHandle = result.itemAt(0).toJavaObject(long.class); - assertFalse(connectionHandle == 0); - - // intentionally escape the context from the lambda - return mainQueryContext; - }); + assertNotEquals(0, connectionHandle); + } - // check the connections map is empty - final int connectionsCount = ModuleUtils.readContextMap(escapedMainQueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, connectionsCount); + // now the query has finished executing and been reset, check the connections map is empty + assertEquals(0, connectionsCountHolder.value.intValue()); transaction.commit(); } diff --git a/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ImplicitConnectionCloseIT.java b/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ImplicitConnectionCloseIT.java index 9ad446a41e..c5bec9c123 100644 --- a/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ImplicitConnectionCloseIT.java +++ b/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/ImplicitConnectionCloseIT.java @@ -32,7 +32,7 @@ */ package org.exist.xquery.modules.sql; -import com.evolvedbinary.j8fu.tuple.Tuple2; +import com.evolvedbinary.j8fu.function.BiConsumerE; import org.exist.EXistException; import org.exist.collections.Collection; import org.exist.security.PermissionDeniedException; @@ -43,14 +43,15 @@ import org.exist.storage.lock.Lock; import org.exist.storage.txn.Txn; import org.exist.test.ExistEmbeddedServer; +import org.exist.util.Holder; import org.exist.util.LockException; import org.exist.util.StringInputSource; import org.exist.xmldb.XmldbURI; import org.exist.xquery.ExternalModule; -import org.exist.xquery.ModuleContext; import org.exist.xquery.XPathException; import org.exist.xquery.XQueryContext; +import org.exist.xquery.XQueryUtil; import org.exist.xquery.modules.ModuleUtils; import org.exist.xquery.value.IntegerValue; import org.exist.xquery.value.Sequence; @@ -74,10 +75,7 @@ import java.util.concurrent.Executor; import java.util.logging.Logger; -import static com.evolvedbinary.j8fu.tuple.Tuple.Tuple; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.exist.xquery.XQueryUtil.executeQuery; -import static org.exist.xquery.XQueryUtil.withCompiledQuery; import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; @@ -135,25 +133,28 @@ public void getJndiConnectionIsAutomaticallyClosed() throws EXistException, XPat try (final DBBroker broker = pool.getBroker(); final Txn transaction = pool.getTransactionManager().beginTransaction()) { - final XQueryContext escapedMainQueryContext = withCompiledQuery(broker, mainQuerySource, mainCompiledQuery -> { - final XQueryContext mainQueryContext = mainCompiledQuery.getContext(); - - // execute the query - final Sequence result = executeQuery(broker, mainCompiledQuery); + // will hold the number of open connections once the query has finished executing + final Holder connectionsCountHolder = new Holder<>(); + final BiConsumerE postExecutionContext = (xqueryContext, result) -> { + final int connectionsCount = ModuleUtils.readContextMap(xqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + connectionsCountHolder.value = connectionsCount; + }; + // execute query + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, mainQuerySource, false, null, null, null, null, postExecutionContext)) { // check that the handle for the sql connection that was created was valid + final Sequence result = queryResult.result; assertEquals(1, result.getItemCount()); assertTrue(result.itemAt(0) instanceof IntegerValue); assertEquals(Type.LONG, result.itemAt(0).getType()); - final long connectionHandle = result.itemAt(0).toJavaObject(long.class); - assertFalse(connectionHandle == 0); - return mainQueryContext; - }); + // check there is an active connection + final long connectionHandle = result.itemAt(0).toJavaObject(long.class); + assertNotEquals(0, connectionHandle); + } - // check the connections map is empty - final int connectionsCount = ModuleUtils.readContextMap(escapedMainQueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, connectionsCount); + // now the query has finished executing and been reset, check the connections map is empty + assertEquals(0, connectionsCountHolder.value.intValue()); // check the connections from our StubDataSource, they should all be closed final Deque createdDataSources = StubDataSourceFactory.CREATED_DATA_SOURCES; @@ -192,42 +193,40 @@ public void getJndiConnectionFromModuleIsAutomaticallyClosed() throws EXistExcep broker.storeDocument(transaction, XmldbURI.create("mymodule.xqm"), new StringInputSource(moduleQuery.getBytes(UTF_8)), xqueryMediaType, collection); } - final Tuple2 escapedContexts = withCompiledQuery(broker, mainQuerySource, mainCompiledQuery -> { - final XQueryContext mainQueryContext = mainCompiledQuery.getContext(); + // will hold the number of open connections in the Main Module once the query has finished executing + final Holder mainModuleConnectionsCountHolder = new Holder<>(); + // will hold the number of open connections in the Library Module once the query has finished executing + final Holder libraryModuleConnectionsCountHolder = new Holder<>(); + final BiConsumerE postExecutionContext = (mainModuleXqueryContext, result) -> { + final int mainModuleConnectionsCount = ModuleUtils.readContextMap(mainModuleXqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + mainModuleConnectionsCountHolder.value = mainModuleConnectionsCount; - // get the context of the library module - final org.exist.xquery.Module[] libraryModules = mainQueryContext.getModules("http://mymodule.com"); - assertEquals(1, libraryModules.length); - assertTrue(libraryModules[0] instanceof ExternalModule); + final org.exist.xquery.Module[] libraryModules = mainModuleXqueryContext.getModules("http://mymodule.com"); final ExternalModule libraryModule = (ExternalModule) libraryModules[0]; - final XQueryContext libraryQueryContext = libraryModule.getContext(); - assertTrue(libraryQueryContext instanceof ModuleContext); + final XQueryContext libraryModuleXqueryContext = libraryModule.getContext(); + final int libraryModuleConnectionsCount = ModuleUtils.readContextMap(libraryModuleXqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + libraryModuleConnectionsCountHolder.value = libraryModuleConnectionsCount; + }; - // execute the query - final Sequence result = executeQuery(broker, mainCompiledQuery); + // execute query + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, mainQuerySource, false, null, null, null, null, postExecutionContext)) { // check that the handle for the sql connection that was created was valid + final Sequence result = queryResult.result; assertEquals(1, result.getItemCount()); assertTrue(result.itemAt(0) instanceof IntegerValue); assertEquals(Type.LONG, result.itemAt(0).getType()); - final long connectionHandle = result.itemAt(0).toJavaObject(long.class); - assertFalse(connectionHandle == 0); - - // intentionally escape the contexts from the lambda - return Tuple(mainQueryContext, (ModuleContext) libraryQueryContext); - }); - final XQueryContext escapedMainQueryContext = escapedContexts._1; - final ModuleContext escapedLibraryQueryContext = escapedContexts._2; - assertTrue(escapedMainQueryContext != escapedLibraryQueryContext); + // check there is an active connection + final long connectionHandle = result.itemAt(0).toJavaObject(long.class); + assertNotEquals(0, connectionHandle); + } - // check the connections were closed in the main module - final int mainConnectionsCount = ModuleUtils.readContextMap(escapedMainQueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, mainConnectionsCount); + // now the query has finished executing and been reset, check the connections map is empty for the Main Module + assertEquals(0, mainModuleConnectionsCountHolder.value.intValue()); - // check the connections were closed in the library module - final int libraryConnectionsCount = ModuleUtils.readContextMap(escapedLibraryQueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, libraryConnectionsCount); + // now the query has finished executing and been reset, check the connections map is empty for the Library Module + assertEquals(0, libraryModuleConnectionsCountHolder.value.intValue()); // check the connections from our StubDataSource, they should all be closed final Deque createdDataSources = StubDataSourceFactory.CREATED_DATA_SOURCES; diff --git a/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/JndiConnectionIT.java b/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/JndiConnectionIT.java index fb50cd2611..49847215b8 100644 --- a/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/JndiConnectionIT.java +++ b/extensions/modules/sql/src/test/java/org/exist/xquery/modules/sql/JndiConnectionIT.java @@ -32,7 +32,7 @@ */ package org.exist.xquery.modules.sql; -import com.evolvedbinary.j8fu.tuple.Tuple2; +import com.evolvedbinary.j8fu.function.BiConsumerE; import org.exist.EXistException; import org.exist.collections.Collection; import org.exist.security.PermissionDeniedException; @@ -43,6 +43,7 @@ import org.exist.storage.lock.Lock; import org.exist.storage.txn.Txn; import org.exist.test.ExistEmbeddedServer; +import org.exist.util.Holder; import org.exist.util.LockException; import org.exist.util.StringInputSource; import org.exist.xmldb.XmldbURI; @@ -69,10 +70,7 @@ import java.util.Optional; import java.util.Properties; -import static com.evolvedbinary.j8fu.tuple.Tuple.Tuple; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.exist.xquery.XQueryUtil.executeQuery; -import static org.exist.xquery.XQueryUtil.withCompiledQuery; import static org.junit.Assert.*; /** @@ -125,25 +123,28 @@ public void getJndiConnectionIsAutomaticallyClosed() throws EXistException, XPat try (final DBBroker broker = pool.getBroker(); final Txn transaction = pool.getTransactionManager().beginTransaction()) { - final XQueryContext escapedMainQueryContext = withCompiledQuery(broker, mainQuerySource, mainCompiledQuery -> { - final XQueryContext mainQueryContext = mainCompiledQuery.getContext(); - - // execute the query - final Sequence result = executeQuery(broker, mainCompiledQuery); + // will hold the number of open connections once the query has finished executing + final Holder connectionsCountHolder = new Holder<>(); + final BiConsumerE postExecutionContext = (xqueryContext, result) -> { + final int connectionsCount = ModuleUtils.readContextMap(xqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + connectionsCountHolder.value = connectionsCount; + }; + // execute query + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, mainQuerySource, false, null, null, null, null, postExecutionContext)) { // check that the handle for the sql connection that was created was valid + final Sequence result = queryResult.result; assertEquals(1, result.getItemCount()); assertTrue(result.itemAt(0) instanceof IntegerValue); assertEquals(Type.LONG, result.itemAt(0).getType()); - final long connectionHandle = result.itemAt(0).toJavaObject(long.class); - assertFalse(connectionHandle == 0); - return mainQueryContext; - }); + // check there is an active connection + final long connectionHandle = result.itemAt(0).toJavaObject(long.class); + assertNotEquals(0, connectionHandle); + } - // check the connections map is empty - final int connectionsCount = ModuleUtils.readContextMap(escapedMainQueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, connectionsCount); + // now the query has finished executing and been reset, check the connections map is empty + assertEquals(0, connectionsCountHolder.value.intValue()); transaction.commit(); } @@ -173,42 +174,41 @@ public void getJndiConnectionFromModuleIsAutomaticallyClosed() throws EXistExcep broker.storeDocument(transaction, XmldbURI.create("mymodule.xqm"), new StringInputSource(moduleQuery.getBytes(UTF_8)), xqueryMediaType, collection); } - final Tuple2 escapedContexts = withCompiledQuery(broker, mainQuerySource, mainCompiledQuery -> { - final XQueryContext mainQueryContext = mainCompiledQuery.getContext(); + // will hold the number of open connections in the Main Module once the query has finished executing + final Holder mainModuleConnectionsCountHolder = new Holder<>(); + // will hold the number of open connections in the Library Module once the query has finished executing + final Holder libraryModuleConnectionsCountHolder = new Holder<>(); + final BiConsumerE postExecutionContext = (mainModuleXqueryContext, result) -> { + final int mainModuleConnectionsCount = ModuleUtils.readContextMap(mainModuleXqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + mainModuleConnectionsCountHolder.value = mainModuleConnectionsCount; - // get the context of the library module - final Module[] libraryModules = mainQueryContext.getModules("http://mymodule.com"); - assertEquals(1, libraryModules.length); - assertTrue(libraryModules[0] instanceof ExternalModule); + final Module[] libraryModules = mainModuleXqueryContext.getModules("http://mymodule.com"); final ExternalModule libraryModule = (ExternalModule) libraryModules[0]; - final XQueryContext libraryQueryContext = libraryModule.getContext(); - assertTrue(libraryQueryContext instanceof ModuleContext); + final XQueryContext libraryModuleXqueryContext = libraryModule.getContext(); + final int libraryModuleConnectionsCount = ModuleUtils.readContextMap(libraryModuleXqueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); + libraryModuleConnectionsCountHolder.value = libraryModuleConnectionsCount; + }; + - // execute the query - final Sequence result = executeQuery(broker, mainCompiledQuery); + // execute query + try (final XQueryUtil.QueryResult queryResult = XQueryUtil.query(broker, mainQuerySource, false, null, null, null, null, postExecutionContext)) { // check that the handle for the sql connection that was created was valid + final Sequence result = queryResult.result; assertEquals(1, result.getItemCount()); assertTrue(result.itemAt(0) instanceof IntegerValue); assertEquals(Type.LONG, result.itemAt(0).getType()); - final long connectionHandle = result.itemAt(0).toJavaObject(long.class); - assertFalse(connectionHandle == 0); - - // intentionally escape the contexts from the lambda - return Tuple(mainQueryContext, (ModuleContext) libraryQueryContext); - }); - final XQueryContext escapedMainQueryContext = escapedContexts._1; - final ModuleContext escapedLibraryQueryContext = escapedContexts._2; - assertTrue(escapedMainQueryContext != escapedLibraryQueryContext); + // check there is an active connection + final long connectionHandle = result.itemAt(0).toJavaObject(long.class); + assertNotEquals(0, connectionHandle); + } - // check the connections were closed in the main module - final int mainConnectionsCount = ModuleUtils.readContextMap(escapedMainQueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, mainConnectionsCount); + // now the query has finished executing and been reset, check the connections map is empty for the Main Module + assertEquals(0, mainModuleConnectionsCountHolder.value.intValue()); - // check the connections were closed in the library module - final int libraryConnectionsCount = ModuleUtils.readContextMap(escapedLibraryQueryContext, SQLModule.CONNECTIONS_CONTEXTVAR, Map::size); - assertEquals(0, libraryConnectionsCount); + // now the query has finished executing and been reset, check the connections map is empty for the Library Module + assertEquals(0, libraryModuleConnectionsCountHolder.value.intValue()); transaction.commit(); } diff --git a/extensions/xqdoc/src/main/java/org/exist/xqdoc/xquery/Scan.java b/extensions/xqdoc/src/main/java/org/exist/xqdoc/xquery/Scan.java index bd943419c9..251954ba3f 100644 --- a/extensions/xqdoc/src/main/java/org/exist/xqdoc/xquery/Scan.java +++ b/extensions/xqdoc/src/main/java/org/exist/xqdoc/xquery/Scan.java @@ -51,12 +51,15 @@ import java.util.regex.Pattern; import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; import org.exist.collections.Collection; -import org.exist.dom.persistent.BinaryDocument; import org.exist.dom.persistent.DocumentImpl; import org.exist.dom.QName; import org.exist.dom.persistent.LockedDocument; import org.exist.security.PermissionDeniedException; -import org.exist.source.*; +import org.exist.source.BinarySource; +import org.exist.source.ClassLoaderSource; +import org.exist.source.DbUriSource; +import org.exist.source.Source; +import org.exist.source.SourceFactory; import org.exist.storage.lock.Lock.LockMode; import org.exist.util.LockException; import org.exist.xmldb.XmldbURI; @@ -159,7 +162,7 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce throw new XPathException(this, "XQuery resource: " + uri + " is not an XQuery or " + "declares a wrong mime-type"); } - source = new DBSource(context.getBroker(), (BinaryDocument) doc, false); + source = DbUriSource.from(context.getBroker().getBrokerPool(), context.getBroker().getCurrentSubject(), doc, false, false); name = doc.getFileURI().toString(); } } catch (LockException e) {